]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py
debian/changelog: Start new release.
[youtubedl] / youtube_dl / __init__.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 __authors__ = (
5 'Ricardo Garcia Gonzalez',
6 'Danny Colligan',
7 'Benjamin Johnson',
8 'Vasyl\' Vavrychuk',
9 'Witold Baryluk',
10 'Paweł Paprota',
11 'Gergely Imreh',
12 'Rogério Brito',
13 'Philipp Hagemeister',
14 'Sören Schulze',
15 'Kevin Ngo',
16 'Ori Avtalion',
17 'shizeeg',
18 'Filippo Valsorda',
19 'Christian Albrecht',
20 'Dave Vasilevsky',
21 'Jaime Marquínez Ferrándiz',
22 'Jeff Crouse',
23 'Osama Khalid',
24 'Michael Walter',
25 'M. Yasoob Ullah Khalid',
26 'Julien Fraichard',
27 'Johny Mo Swag',
28 'Axel Noack',
29 'Albert Kim',
30 'Pierre Rudloff',
31 'Huarong Huo',
32 'Ismael Mejía',
33 'Steffan \'Ruirize\' James',
34 'Andras Elso',
35 'Jelle van der Waa',
36 'Marcin Cieślak',
37 'Anton Larionov',
38 'Takuya Tsuchida',
39 'Sergey M.',
40 'Michael Orlitzky',
41 'Chris Gahan',
42 'Saimadhav Heblikar',
43 'Mike Col',
44 'Oleg Prutz',
45 'pulpe',
46 'Andreas Schmitz',
47 'Michael Kaiser',
48 'Niklas Laxström',
49 'David Triendl',
50 'Anthony Weems',
51 'David Wagner',
52 'Juan C. Olivares',
53 'Mattias Harrysson',
54 'phaer',
55 'Sainyam Kapoor',
56 'Nicolas Évrard',
57 'Jason Normore',
58 'Hoje Lee',
59 'Adam Thalhammer',
60 'Georg Jähnig',
61 'Ralf Haring',
62 'Koki Takahashi',
63 'Ariset Llerena',
64 'Adam Malcontenti-Wilson',
65 'Tobias Bell',
66 'Naglis Jonaitis',
67 )
68
69 __license__ = 'Public Domain'
70
71 import codecs
72 import io
73 import locale
74 import optparse
75 import os
76 import random
77 import re
78 import shlex
79 import sys
80
81
82 from .utils import (
83 compat_getpass,
84 compat_print,
85 DateRange,
86 DEFAULT_OUTTMPL,
87 decodeOption,
88 get_term_width,
89 DownloadError,
90 get_cachedir,
91 MaxDownloadsReached,
92 preferredencoding,
93 read_batch_urls,
94 SameFileError,
95 setproctitle,
96 std_headers,
97 write_string,
98 )
99 from .update import update_self
100 from .FileDownloader import (
101 FileDownloader,
102 )
103 from .extractor import gen_extractors
104 from .version import __version__
105 from .YoutubeDL import YoutubeDL
106 from .postprocessor import (
107 AtomicParsleyPP,
108 FFmpegAudioFixPP,
109 FFmpegMetadataPP,
110 FFmpegVideoConvertor,
111 FFmpegExtractAudioPP,
112 FFmpegEmbedSubtitlePP,
113 XAttrMetadataPP,
114 )
115
116
117 def parseOpts(overrideArguments=None):
118 def _readOptions(filename_bytes, default=[]):
119 try:
120 optionf = open(filename_bytes)
121 except IOError:
122 return default # silently skip if file is not present
123 try:
124 res = []
125 for l in optionf:
126 res += shlex.split(l, comments=True)
127 finally:
128 optionf.close()
129 return res
130
131 def _readUserConf():
132 xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
133 if xdg_config_home:
134 userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
135 if not os.path.isfile(userConfFile):
136 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
137 else:
138 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
139 if not os.path.isfile(userConfFile):
140 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
141 userConf = _readOptions(userConfFile, None)
142
143 if userConf is None:
144 appdata_dir = os.environ.get('appdata')
145 if appdata_dir:
146 userConf = _readOptions(
147 os.path.join(appdata_dir, 'youtube-dl', 'config'),
148 default=None)
149 if userConf is None:
150 userConf = _readOptions(
151 os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
152 default=None)
153
154 if userConf is None:
155 userConf = _readOptions(
156 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
157 default=None)
158 if userConf is None:
159 userConf = _readOptions(
160 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
161 default=None)
162
163 if userConf is None:
164 userConf = []
165
166 return userConf
167
168 def _format_option_string(option):
169 ''' ('-o', '--option') -> -o, --format METAVAR'''
170
171 opts = []
172
173 if option._short_opts:
174 opts.append(option._short_opts[0])
175 if option._long_opts:
176 opts.append(option._long_opts[0])
177 if len(opts) > 1:
178 opts.insert(1, ', ')
179
180 if option.takes_value(): opts.append(' %s' % option.metavar)
181
182 return "".join(opts)
183
184 def _comma_separated_values_options_callback(option, opt_str, value, parser):
185 setattr(parser.values, option.dest, value.split(','))
186
187 def _hide_login_info(opts):
188 opts = list(opts)
189 for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
190 try:
191 i = opts.index(private_opt)
192 opts[i+1] = '<PRIVATE>'
193 except ValueError:
194 pass
195 return opts
196
197 max_width = 80
198 max_help_position = 80
199
200 # No need to wrap help messages if we're on a wide console
201 columns = get_term_width()
202 if columns: max_width = columns
203
204 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
205 fmt.format_option_strings = _format_option_string
206
207 kw = {
208 'version' : __version__,
209 'formatter' : fmt,
210 'usage' : '%prog [options] url [url...]',
211 'conflict_handler' : 'resolve',
212 }
213
214 parser = optparse.OptionParser(**kw)
215
216 # option groups
217 general = optparse.OptionGroup(parser, 'General Options')
218 selection = optparse.OptionGroup(parser, 'Video Selection')
219 authentication = optparse.OptionGroup(parser, 'Authentication Options')
220 video_format = optparse.OptionGroup(parser, 'Video Format Options')
221 subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
222 downloader = optparse.OptionGroup(parser, 'Download Options')
223 postproc = optparse.OptionGroup(parser, 'Post-processing Options')
224 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
225 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
226
227 general.add_option('-h', '--help',
228 action='help', help='print this help text and exit')
229 general.add_option('-v', '--version',
230 action='version', help='print program version and exit')
231 general.add_option('-U', '--update',
232 action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
233 general.add_option('-i', '--ignore-errors',
234 action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
235 general.add_option('--abort-on-error',
236 action='store_false', dest='ignoreerrors',
237 help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
238 general.add_option('--dump-user-agent',
239 action='store_true', dest='dump_user_agent',
240 help='display the current browser identification', default=False)
241 general.add_option('--user-agent',
242 dest='user_agent', help='specify a custom user agent', metavar='UA')
243 general.add_option('--referer',
244 dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
245 metavar='REF', default=None)
246 general.add_option('--add-header',
247 dest='headers', help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', action="append",
248 metavar='FIELD:VALUE')
249 general.add_option('--list-extractors',
250 action='store_true', dest='list_extractors',
251 help='List all supported extractors and the URLs they would handle', default=False)
252 general.add_option('--extractor-descriptions',
253 action='store_true', dest='list_extractor_descriptions',
254 help='Output descriptions of all supported extractors', default=False)
255 general.add_option(
256 '--proxy', dest='proxy', default=None, metavar='URL',
257 help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
258 general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
259 general.add_option(
260 '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
261 help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
262 general.add_option(
263 '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
264 help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
265 general.add_option(
266 '--no-cache-dir', action='store_const', const=None, dest='cachedir',
267 help='Disable filesystem caching')
268 general.add_option(
269 '--socket-timeout', dest='socket_timeout',
270 type=float, default=None, help=u'Time to wait before giving up, in seconds')
271 general.add_option(
272 '--bidi-workaround', dest='bidi_workaround', action='store_true',
273 help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
274 general.add_option(
275 '--default-search',
276 dest='default_search', metavar='PREFIX',
277 help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess. The default value "error" just throws an error.')
278 general.add_option(
279 '--ignore-config',
280 action='store_true',
281 help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
282 general.add_option(
283 '--encoding', dest='encoding', metavar='ENCODING',
284 help='Force the specified encoding (experimental)')
285
286 selection.add_option(
287 '--playlist-start',
288 dest='playliststart', metavar='NUMBER', default=1, type=int,
289 help='playlist video to start at (default is %default)')
290 selection.add_option(
291 '--playlist-end',
292 dest='playlistend', metavar='NUMBER', default=None, type=int,
293 help='playlist video to end at (default is last)')
294 selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
295 selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
296 selection.add_option('--max-downloads', metavar='NUMBER',
297 dest='max_downloads', type=int, default=None,
298 help='Abort after downloading NUMBER files')
299 selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
300 selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
301 selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
302 selection.add_option(
303 '--datebefore', metavar='DATE', dest='datebefore', default=None,
304 help='download only videos uploaded on or before this date (i.e. inclusive)')
305 selection.add_option(
306 '--dateafter', metavar='DATE', dest='dateafter', default=None,
307 help='download only videos uploaded on or after this date (i.e. inclusive)')
308 selection.add_option(
309 '--min-views', metavar='COUNT', dest='min_views',
310 default=None, type=int,
311 help="Do not download any videos with less than COUNT views",)
312 selection.add_option(
313 '--max-views', metavar='COUNT', dest='max_views',
314 default=None, type=int,
315 help="Do not download any videos with more than COUNT views",)
316 selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
317 selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
318 help='download only videos suitable for the given age',
319 default=None, type=int)
320 selection.add_option('--download-archive', metavar='FILE',
321 dest='download_archive',
322 help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
323 selection.add_option(
324 '--include-ads', dest='include_ads',
325 action='store_true',
326 help='Download advertisements as well (experimental)')
327 selection.add_option(
328 '--youtube-include-dash-manifest', action='store_true',
329 dest='youtube_include_dash_manifest', default=False,
330 help='Try to download the DASH manifest on YouTube videos (experimental)')
331
332 authentication.add_option('-u', '--username',
333 dest='username', metavar='USERNAME', help='account username')
334 authentication.add_option('-p', '--password',
335 dest='password', metavar='PASSWORD', help='account password')
336 authentication.add_option('-n', '--netrc',
337 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
338 authentication.add_option('--video-password',
339 dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
340
341
342 video_format.add_option('-f', '--format',
343 action='store', dest='format', metavar='FORMAT', default=None,
344 help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
345 video_format.add_option('--all-formats',
346 action='store_const', dest='format', help='download all available video formats', const='all')
347 video_format.add_option('--prefer-free-formats',
348 action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
349 video_format.add_option('--max-quality',
350 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
351 video_format.add_option('-F', '--list-formats',
352 action='store_true', dest='listformats', help='list all available formats')
353
354 subtitles.add_option('--write-sub', '--write-srt',
355 action='store_true', dest='writesubtitles',
356 help='write subtitle file', default=False)
357 subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
358 action='store_true', dest='writeautomaticsub',
359 help='write automatic subtitle file (youtube only)', default=False)
360 subtitles.add_option('--all-subs',
361 action='store_true', dest='allsubtitles',
362 help='downloads all the available subtitles of the video', default=False)
363 subtitles.add_option('--list-subs',
364 action='store_true', dest='listsubtitles',
365 help='lists all available subtitles for the video', default=False)
366 subtitles.add_option('--sub-format',
367 action='store', dest='subtitlesformat', metavar='FORMAT',
368 help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
369 subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
370 action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
371 default=[], callback=_comma_separated_values_options_callback,
372 help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
373
374 downloader.add_option('-r', '--rate-limit',
375 dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
376 downloader.add_option('-R', '--retries',
377 dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
378 downloader.add_option('--buffer-size',
379 dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
380 downloader.add_option('--no-resize-buffer',
381 action='store_true', dest='noresizebuffer',
382 help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
383 downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
384
385 verbosity.add_option('-q', '--quiet',
386 action='store_true', dest='quiet', help='activates quiet mode', default=False)
387 verbosity.add_option(
388 '--no-warnings',
389 dest='no_warnings', action='store_true', default=False,
390 help='Ignore warnings')
391 verbosity.add_option('-s', '--simulate',
392 action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
393 verbosity.add_option('--skip-download',
394 action='store_true', dest='skip_download', help='do not download the video', default=False)
395 verbosity.add_option('-g', '--get-url',
396 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
397 verbosity.add_option('-e', '--get-title',
398 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
399 verbosity.add_option('--get-id',
400 action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
401 verbosity.add_option('--get-thumbnail',
402 action='store_true', dest='getthumbnail',
403 help='simulate, quiet but print thumbnail URL', default=False)
404 verbosity.add_option('--get-description',
405 action='store_true', dest='getdescription',
406 help='simulate, quiet but print video description', default=False)
407 verbosity.add_option('--get-duration',
408 action='store_true', dest='getduration',
409 help='simulate, quiet but print video length', default=False)
410 verbosity.add_option('--get-filename',
411 action='store_true', dest='getfilename',
412 help='simulate, quiet but print output filename', default=False)
413 verbosity.add_option('--get-format',
414 action='store_true', dest='getformat',
415 help='simulate, quiet but print output format', default=False)
416 verbosity.add_option('-j', '--dump-json',
417 action='store_true', dest='dumpjson',
418 help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
419 verbosity.add_option('--newline',
420 action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
421 verbosity.add_option('--no-progress',
422 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
423 verbosity.add_option('--console-title',
424 action='store_true', dest='consoletitle',
425 help='display progress in console titlebar', default=False)
426 verbosity.add_option('-v', '--verbose',
427 action='store_true', dest='verbose', help='print various debugging information', default=False)
428 verbosity.add_option('--dump-intermediate-pages',
429 action='store_true', dest='dump_intermediate_pages', default=False,
430 help='print downloaded pages to debug problems (very verbose)')
431 verbosity.add_option('--write-pages',
432 action='store_true', dest='write_pages', default=False,
433 help='Write downloaded intermediary pages to files in the current directory to debug problems')
434 verbosity.add_option('--youtube-print-sig-code',
435 action='store_true', dest='youtube_print_sig_code', default=False,
436 help=optparse.SUPPRESS_HELP)
437 verbosity.add_option('--print-traffic',
438 dest='debug_printtraffic', action='store_true', default=False,
439 help='Display sent and read HTTP traffic')
440
441
442 filesystem.add_option('-t', '--title',
443 action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
444 filesystem.add_option('--id',
445 action='store_true', dest='useid', help='use only video ID in file name', default=False)
446 filesystem.add_option('-l', '--literal',
447 action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
448 filesystem.add_option('-A', '--auto-number',
449 action='store_true', dest='autonumber',
450 help='number downloaded files starting from 00000', default=False)
451 filesystem.add_option('-o', '--output',
452 dest='outtmpl', metavar='TEMPLATE',
453 help=('output filename template. Use %(title)s to get the title, '
454 '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
455 '%(autonumber)s to get an automatically incremented number, '
456 '%(ext)s for the filename extension, '
457 '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
458 '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
459 '%(upload_date)s for the upload date (YYYYMMDD), '
460 '%(extractor)s for the provider (youtube, metacafe, etc), '
461 '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
462 '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
463 '%(height)s and %(width)s for the width and height of the video format. '
464 '%(resolution)s for a textual description of the resolution of the video format. '
465 'Use - to output to stdout. Can also be used to download to a different directory, '
466 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
467 filesystem.add_option('--autonumber-size',
468 dest='autonumber_size', metavar='NUMBER',
469 help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
470 filesystem.add_option('--restrict-filenames',
471 action='store_true', dest='restrictfilenames',
472 help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
473 filesystem.add_option('-a', '--batch-file',
474 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
475 filesystem.add_option('--load-info',
476 dest='load_info_filename', metavar='FILE',
477 help='json file containing the video information (created with the "--write-json" option)')
478 filesystem.add_option('-w', '--no-overwrites',
479 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
480 filesystem.add_option('-c', '--continue',
481 action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
482 filesystem.add_option('--no-continue',
483 action='store_false', dest='continue_dl',
484 help='do not resume partially downloaded files (restart from beginning)')
485 filesystem.add_option('--cookies',
486 dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
487 filesystem.add_option('--no-part',
488 action='store_true', dest='nopart', help='do not use .part files', default=False)
489 filesystem.add_option('--no-mtime',
490 action='store_false', dest='updatetime',
491 help='do not use the Last-modified header to set the file modification time', default=True)
492 filesystem.add_option('--write-description',
493 action='store_true', dest='writedescription',
494 help='write video description to a .description file', default=False)
495 filesystem.add_option('--write-info-json',
496 action='store_true', dest='writeinfojson',
497 help='write video metadata to a .info.json file', default=False)
498 filesystem.add_option('--write-annotations',
499 action='store_true', dest='writeannotations',
500 help='write video annotations to a .annotation file', default=False)
501 filesystem.add_option('--write-thumbnail',
502 action='store_true', dest='writethumbnail',
503 help='write thumbnail image to disk', default=False)
504
505
506 postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
507 help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
508 postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
509 help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
510 postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
511 help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
512 postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
513 help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
514 postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
515 help='keeps the video file on disk after the post-processing; the video is erased by default')
516 postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
517 help='do not overwrite post-processed files; the post-processed files are overwritten by default')
518 postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
519 help='embed subtitles in the video (only for mp4 videos)')
520 postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
521 help='embed thumbnail in the audio as cover art')
522 postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
523 help='write metadata to the video file')
524 postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
525 help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
526 postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
527 help='Prefer avconv over ffmpeg for running the postprocessors (default)')
528 postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
529 help='Prefer ffmpeg over avconv for running the postprocessors')
530
531
532 parser.add_option_group(general)
533 parser.add_option_group(selection)
534 parser.add_option_group(downloader)
535 parser.add_option_group(filesystem)
536 parser.add_option_group(verbosity)
537 parser.add_option_group(video_format)
538 parser.add_option_group(subtitles)
539 parser.add_option_group(authentication)
540 parser.add_option_group(postproc)
541
542 if overrideArguments is not None:
543 opts, args = parser.parse_args(overrideArguments)
544 if opts.verbose:
545 write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
546 else:
547 commandLineConf = sys.argv[1:]
548 if '--ignore-config' in commandLineConf:
549 systemConf = []
550 userConf = []
551 else:
552 systemConf = _readOptions('/etc/youtube-dl.conf')
553 if '--ignore-config' in systemConf:
554 userConf = []
555 else:
556 userConf = _readUserConf()
557 argv = systemConf + userConf + commandLineConf
558
559 opts, args = parser.parse_args(argv)
560 if opts.verbose:
561 write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
562 write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
563 write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
564
565 return parser, opts, args
566
567
568 def _real_main(argv=None):
569 # Compatibility fixes for Windows
570 if sys.platform == 'win32':
571 # https://github.com/rg3/youtube-dl/issues/820
572 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
573
574 setproctitle(u'youtube-dl')
575
576 parser, opts, args = parseOpts(argv)
577
578 # Set user agent
579 if opts.user_agent is not None:
580 std_headers['User-Agent'] = opts.user_agent
581
582 # Set referer
583 if opts.referer is not None:
584 std_headers['Referer'] = opts.referer
585
586 # Custom HTTP headers
587 if opts.headers is not None:
588 for h in opts.headers:
589 if h.find(':', 1) < 0:
590 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
591 key, value = h.split(':', 2)
592 if opts.verbose:
593 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
594 std_headers[key] = value
595
596 # Dump user agent
597 if opts.dump_user_agent:
598 compat_print(std_headers['User-Agent'])
599 sys.exit(0)
600
601 # Batch file verification
602 batch_urls = []
603 if opts.batchfile is not None:
604 try:
605 if opts.batchfile == '-':
606 batchfd = sys.stdin
607 else:
608 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
609 batch_urls = read_batch_urls(batchfd)
610 if opts.verbose:
611 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
612 except IOError:
613 sys.exit(u'ERROR: batch file could not be read')
614 all_urls = batch_urls + args
615 all_urls = [url.strip() for url in all_urls]
616 _enc = preferredencoding()
617 all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
618
619 extractors = gen_extractors()
620
621 if opts.list_extractors:
622 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
623 compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
624 matchedUrls = [url for url in all_urls if ie.suitable(url)]
625 for mu in matchedUrls:
626 compat_print(u' ' + mu)
627 sys.exit(0)
628 if opts.list_extractor_descriptions:
629 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
630 if not ie._WORKING:
631 continue
632 desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
633 if desc is False:
634 continue
635 if hasattr(ie, 'SEARCH_KEY'):
636 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
637 _COUNTS = (u'', u'5', u'10', u'all')
638 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
639 compat_print(desc)
640 sys.exit(0)
641
642
643 # Conflicting, missing and erroneous options
644 if opts.usenetrc and (opts.username is not None or opts.password is not None):
645 parser.error(u'using .netrc conflicts with giving username/password')
646 if opts.password is not None and opts.username is None:
647 parser.error(u'account username missing\n')
648 if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
649 parser.error(u'using output template conflicts with using title, video ID or auto number')
650 if opts.usetitle and opts.useid:
651 parser.error(u'using title conflicts with using video ID')
652 if opts.username is not None and opts.password is None:
653 opts.password = compat_getpass(u'Type account password and press [Return]: ')
654 if opts.ratelimit is not None:
655 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
656 if numeric_limit is None:
657 parser.error(u'invalid rate limit specified')
658 opts.ratelimit = numeric_limit
659 if opts.min_filesize is not None:
660 numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
661 if numeric_limit is None:
662 parser.error(u'invalid min_filesize specified')
663 opts.min_filesize = numeric_limit
664 if opts.max_filesize is not None:
665 numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
666 if numeric_limit is None:
667 parser.error(u'invalid max_filesize specified')
668 opts.max_filesize = numeric_limit
669 if opts.retries is not None:
670 try:
671 opts.retries = int(opts.retries)
672 except (TypeError, ValueError):
673 parser.error(u'invalid retry count specified')
674 if opts.buffersize is not None:
675 numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
676 if numeric_buffersize is None:
677 parser.error(u'invalid buffer size specified')
678 opts.buffersize = numeric_buffersize
679 if opts.playliststart <= 0:
680 raise ValueError(u'Playlist start must be positive')
681 if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
682 raise ValueError(u'Playlist end must be greater than playlist start')
683 if opts.extractaudio:
684 if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
685 parser.error(u'invalid audio format specified')
686 if opts.audioquality:
687 opts.audioquality = opts.audioquality.strip('k').strip('K')
688 if not opts.audioquality.isdigit():
689 parser.error(u'invalid audio quality specified')
690 if opts.recodevideo is not None:
691 if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
692 parser.error(u'invalid video recode format specified')
693 if opts.date is not None:
694 date = DateRange.day(opts.date)
695 else:
696 date = DateRange(opts.dateafter, opts.datebefore)
697 if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
698 parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
699
700 # Do not download videos when there are audio-only formats
701 if opts.extractaudio and not opts.keepvideo and opts.format is None:
702 opts.format = 'bestaudio/best'
703
704 # --all-sub automatically sets --write-sub if --write-auto-sub is not given
705 # this was the old behaviour if only --all-sub was given.
706 if opts.allsubtitles and (opts.writeautomaticsub == False):
707 opts.writesubtitles = True
708
709 if sys.version_info < (3,):
710 # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
711 if opts.outtmpl is not None:
712 opts.outtmpl = opts.outtmpl.decode(preferredencoding())
713 outtmpl =((opts.outtmpl is not None and opts.outtmpl)
714 or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
715 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
716 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
717 or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
718 or (opts.useid and u'%(id)s.%(ext)s')
719 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
720 or DEFAULT_OUTTMPL)
721 if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
722 parser.error(u'Cannot download a video and extract audio into the same'
723 u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
724 u' template'.format(outtmpl))
725
726 any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
727 download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
728
729 ydl_opts = {
730 'usenetrc': opts.usenetrc,
731 'username': opts.username,
732 'password': opts.password,
733 'videopassword': opts.videopassword,
734 'quiet': (opts.quiet or any_printing),
735 'no_warnings': opts.no_warnings,
736 'forceurl': opts.geturl,
737 'forcetitle': opts.gettitle,
738 'forceid': opts.getid,
739 'forcethumbnail': opts.getthumbnail,
740 'forcedescription': opts.getdescription,
741 'forceduration': opts.getduration,
742 'forcefilename': opts.getfilename,
743 'forceformat': opts.getformat,
744 'forcejson': opts.dumpjson,
745 'simulate': opts.simulate,
746 'skip_download': (opts.skip_download or opts.simulate or any_printing),
747 'format': opts.format,
748 'format_limit': opts.format_limit,
749 'listformats': opts.listformats,
750 'outtmpl': outtmpl,
751 'autonumber_size': opts.autonumber_size,
752 'restrictfilenames': opts.restrictfilenames,
753 'ignoreerrors': opts.ignoreerrors,
754 'ratelimit': opts.ratelimit,
755 'nooverwrites': opts.nooverwrites,
756 'retries': opts.retries,
757 'buffersize': opts.buffersize,
758 'noresizebuffer': opts.noresizebuffer,
759 'continuedl': opts.continue_dl,
760 'noprogress': opts.noprogress,
761 'progress_with_newline': opts.progress_with_newline,
762 'playliststart': opts.playliststart,
763 'playlistend': opts.playlistend,
764 'noplaylist': opts.noplaylist,
765 'logtostderr': opts.outtmpl == '-',
766 'consoletitle': opts.consoletitle,
767 'nopart': opts.nopart,
768 'updatetime': opts.updatetime,
769 'writedescription': opts.writedescription,
770 'writeannotations': opts.writeannotations,
771 'writeinfojson': opts.writeinfojson,
772 'writethumbnail': opts.writethumbnail,
773 'writesubtitles': opts.writesubtitles,
774 'writeautomaticsub': opts.writeautomaticsub,
775 'allsubtitles': opts.allsubtitles,
776 'listsubtitles': opts.listsubtitles,
777 'subtitlesformat': opts.subtitlesformat,
778 'subtitleslangs': opts.subtitleslangs,
779 'matchtitle': decodeOption(opts.matchtitle),
780 'rejecttitle': decodeOption(opts.rejecttitle),
781 'max_downloads': opts.max_downloads,
782 'prefer_free_formats': opts.prefer_free_formats,
783 'verbose': opts.verbose,
784 'dump_intermediate_pages': opts.dump_intermediate_pages,
785 'write_pages': opts.write_pages,
786 'test': opts.test,
787 'keepvideo': opts.keepvideo,
788 'min_filesize': opts.min_filesize,
789 'max_filesize': opts.max_filesize,
790 'min_views': opts.min_views,
791 'max_views': opts.max_views,
792 'daterange': date,
793 'cachedir': opts.cachedir,
794 'youtube_print_sig_code': opts.youtube_print_sig_code,
795 'age_limit': opts.age_limit,
796 'download_archive': download_archive_fn,
797 'cookiefile': opts.cookiefile,
798 'nocheckcertificate': opts.no_check_certificate,
799 'prefer_insecure': opts.prefer_insecure,
800 'proxy': opts.proxy,
801 'socket_timeout': opts.socket_timeout,
802 'bidi_workaround': opts.bidi_workaround,
803 'debug_printtraffic': opts.debug_printtraffic,
804 'prefer_ffmpeg': opts.prefer_ffmpeg,
805 'include_ads': opts.include_ads,
806 'default_search': opts.default_search,
807 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
808 'encoding': opts.encoding,
809 }
810
811 with YoutubeDL(ydl_opts) as ydl:
812 ydl.print_debug_header()
813 ydl.add_default_info_extractors()
814
815 # PostProcessors
816 # Add the metadata pp first, the other pps will copy it
817 if opts.addmetadata:
818 ydl.add_post_processor(FFmpegMetadataPP())
819 if opts.extractaudio:
820 ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
821 if opts.recodevideo:
822 ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
823 if opts.embedsubtitles:
824 ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
825 if opts.xattrs:
826 ydl.add_post_processor(XAttrMetadataPP())
827 if opts.embedthumbnail:
828 if not opts.addmetadata:
829 ydl.add_post_processor(FFmpegAudioFixPP())
830 ydl.add_post_processor(AtomicParsleyPP())
831
832 # Update version
833 if opts.update_self:
834 update_self(ydl.to_screen, opts.verbose)
835
836 # Maybe do nothing
837 if (len(all_urls) < 1) and (opts.load_info_filename is None):
838 if not opts.update_self:
839 parser.error(u'you must provide at least one URL')
840 else:
841 sys.exit()
842
843 try:
844 if opts.load_info_filename is not None:
845 retcode = ydl.download_with_info_file(opts.load_info_filename)
846 else:
847 retcode = ydl.download(all_urls)
848 except MaxDownloadsReached:
849 ydl.to_screen(u'--max-download limit reached, aborting.')
850 retcode = 101
851
852 sys.exit(retcode)
853
854
855 def main(argv=None):
856 try:
857 _real_main(argv)
858 except DownloadError:
859 sys.exit(1)
860 except SameFileError:
861 sys.exit(u'ERROR: fixed output name but more than one file to download')
862 except KeyboardInterrupt:
863 sys.exit(u'\nERROR: Interrupted by user')