]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py
Update changelog.
[youtubedl] / youtube_dl / __init__.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 __authors__ = (
5 'Ricardo Garcia Gonzalez',
6 'Danny Colligan',
7 'Benjamin Johnson',
8 'Vasyl\' Vavrychuk',
9 'Witold Baryluk',
10 'Paweł Paprota',
11 'Gergely Imreh',
12 'Rogério Brito',
13 'Philipp Hagemeister',
14 'Sören Schulze',
15 'Kevin Ngo',
16 'Ori Avtalion',
17 'shizeeg',
18 'Filippo Valsorda',
19 'Christian Albrecht',
20 'Dave Vasilevsky',
21 'Jaime Marquínez Ferrándiz',
22 'Jeff Crouse',
23 'Osama Khalid',
24 'Michael Walter',
25 'M. Yasoob Ullah Khalid',
26 'Julien Fraichard',
27 'Johny Mo Swag',
28 'Axel Noack',
29 'Albert Kim',
30 'Pierre Rudloff',
31 'Huarong Huo',
32 'Ismael Mejía',
33 'Steffan \'Ruirize\' James',
34 'Andras Elso',
35 'Jelle van der Waa',
36 'Marcin Cieślak',
37 'Anton Larionov',
38 'Takuya Tsuchida',
39 'Sergey M.',
40 'Michael Orlitzky',
41 'Chris Gahan',
42 'Saimadhav Heblikar',
43 'Mike Col',
44 'Oleg Prutz',
45 'pulpe',
46 'Andreas Schmitz',
47 'Michael Kaiser',
48 'Niklas Laxström',
49 )
50
51 __license__ = 'Public Domain'
52
53 import codecs
54 import getpass
55 import locale
56 import optparse
57 import os
58 import random
59 import re
60 import shlex
61 import sys
62
63
64 from .utils import (
65 compat_print,
66 DateRange,
67 decodeOption,
68 get_term_width,
69 DownloadError,
70 get_cachedir,
71 MaxDownloadsReached,
72 preferredencoding,
73 SameFileError,
74 setproctitle,
75 std_headers,
76 write_string,
77 )
78 from .update import update_self
79 from .FileDownloader import (
80 FileDownloader,
81 )
82 from .extractor import gen_extractors
83 from .version import __version__
84 from .YoutubeDL import YoutubeDL
85 from .postprocessor import (
86 FFmpegMetadataPP,
87 FFmpegVideoConvertor,
88 FFmpegExtractAudioPP,
89 FFmpegEmbedSubtitlePP,
90 XAttrMetadataPP,
91 )
92
93
94 def parseOpts(overrideArguments=None):
95 def _readOptions(filename_bytes, default=[]):
96 try:
97 optionf = open(filename_bytes)
98 except IOError:
99 return default # silently skip if file is not present
100 try:
101 res = []
102 for l in optionf:
103 res += shlex.split(l, comments=True)
104 finally:
105 optionf.close()
106 return res
107
108 def _readUserConf():
109 xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
110 if xdg_config_home:
111 userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
112 if not os.path.isfile(userConfFile):
113 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
114 else:
115 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
116 if not os.path.isfile(userConfFile):
117 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
118 userConf = _readOptions(userConfFile, None)
119
120 if userConf is None:
121 appdata_dir = os.environ.get('appdata')
122 if appdata_dir:
123 userConf = _readOptions(
124 os.path.join(appdata_dir, 'youtube-dl', 'config'),
125 default=None)
126 if userConf is None:
127 userConf = _readOptions(
128 os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
129 default=None)
130
131 if userConf is None:
132 userConf = _readOptions(
133 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
134 default=None)
135 if userConf is None:
136 userConf = _readOptions(
137 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
138 default=None)
139
140 if userConf is None:
141 userConf = []
142
143 return userConf
144
145 def _format_option_string(option):
146 ''' ('-o', '--option') -> -o, --format METAVAR'''
147
148 opts = []
149
150 if option._short_opts:
151 opts.append(option._short_opts[0])
152 if option._long_opts:
153 opts.append(option._long_opts[0])
154 if len(opts) > 1:
155 opts.insert(1, ', ')
156
157 if option.takes_value(): opts.append(' %s' % option.metavar)
158
159 return "".join(opts)
160
161 def _comma_separated_values_options_callback(option, opt_str, value, parser):
162 setattr(parser.values, option.dest, value.split(','))
163
164 def _hide_login_info(opts):
165 opts = list(opts)
166 for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
167 try:
168 i = opts.index(private_opt)
169 opts[i+1] = '<PRIVATE>'
170 except ValueError:
171 pass
172 return opts
173
174 max_width = 80
175 max_help_position = 80
176
177 # No need to wrap help messages if we're on a wide console
178 columns = get_term_width()
179 if columns: max_width = columns
180
181 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
182 fmt.format_option_strings = _format_option_string
183
184 kw = {
185 'version' : __version__,
186 'formatter' : fmt,
187 'usage' : '%prog [options] url [url...]',
188 'conflict_handler' : 'resolve',
189 }
190
191 parser = optparse.OptionParser(**kw)
192
193 # option groups
194 general = optparse.OptionGroup(parser, 'General Options')
195 selection = optparse.OptionGroup(parser, 'Video Selection')
196 authentication = optparse.OptionGroup(parser, 'Authentication Options')
197 video_format = optparse.OptionGroup(parser, 'Video Format Options')
198 subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
199 downloader = optparse.OptionGroup(parser, 'Download Options')
200 postproc = optparse.OptionGroup(parser, 'Post-processing Options')
201 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
202 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
203
204 general.add_option('-h', '--help',
205 action='help', help='print this help text and exit')
206 general.add_option('-v', '--version',
207 action='version', help='print program version and exit')
208 general.add_option('-U', '--update',
209 action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
210 general.add_option('-i', '--ignore-errors',
211 action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
212 general.add_option('--abort-on-error',
213 action='store_false', dest='ignoreerrors',
214 help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
215 general.add_option('--dump-user-agent',
216 action='store_true', dest='dump_user_agent',
217 help='display the current browser identification', default=False)
218 general.add_option('--user-agent',
219 dest='user_agent', help='specify a custom user agent', metavar='UA')
220 general.add_option('--referer',
221 dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
222 metavar='REF', default=None)
223 general.add_option('--list-extractors',
224 action='store_true', dest='list_extractors',
225 help='List all supported extractors and the URLs they would handle', default=False)
226 general.add_option('--extractor-descriptions',
227 action='store_true', dest='list_extractor_descriptions',
228 help='Output descriptions of all supported extractors', default=False)
229 general.add_option(
230 '--proxy', dest='proxy', default=None, metavar='URL',
231 help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
232 general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
233 general.add_option(
234 '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
235 help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
236 general.add_option(
237 '--no-cache-dir', action='store_const', const=None, dest='cachedir',
238 help='Disable filesystem caching')
239 general.add_option(
240 '--socket-timeout', dest='socket_timeout',
241 type=float, default=None, help=u'Time to wait before giving up, in seconds')
242 general.add_option(
243 '--bidi-workaround', dest='bidi_workaround', action='store_true',
244 help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
245 general.add_option('--default-search',
246 dest='default_search', metavar='PREFIX',
247 help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". By default (with value "auto") youtube-dl guesses.')
248 general.add_option(
249 '--ignore-config',
250 action='store_true',
251 help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
252
253
254 selection.add_option(
255 '--playlist-start',
256 dest='playliststart', metavar='NUMBER', default=1, type=int,
257 help='playlist video to start at (default is %default)')
258 selection.add_option(
259 '--playlist-end',
260 dest='playlistend', metavar='NUMBER', default=None, type=int,
261 help='playlist video to end at (default is last)')
262 selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
263 selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
264 selection.add_option('--max-downloads', metavar='NUMBER',
265 dest='max_downloads', type=int, default=None,
266 help='Abort after downloading NUMBER files')
267 selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
268 selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
269 selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
270 selection.add_option(
271 '--datebefore', metavar='DATE', dest='datebefore', default=None,
272 help='download only videos uploaded on or before this date (i.e. inclusive)')
273 selection.add_option(
274 '--dateafter', metavar='DATE', dest='dateafter', default=None,
275 help='download only videos uploaded on or after this date (i.e. inclusive)')
276 selection.add_option(
277 '--min-views', metavar='COUNT', dest='min_views',
278 default=None, type=int,
279 help="Do not download any videos with less than COUNT views",)
280 selection.add_option(
281 '--max-views', metavar='COUNT', dest='max_views',
282 default=None, type=int,
283 help="Do not download any videos with more than COUNT views",)
284 selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
285 selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
286 help='download only videos suitable for the given age',
287 default=None, type=int)
288 selection.add_option('--download-archive', metavar='FILE',
289 dest='download_archive',
290 help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
291 selection.add_option(
292 '--include-ads', dest='include_ads',
293 action='store_true',
294 help='Download advertisements as well (experimental)')
295 selection.add_option(
296 '--youtube-include-dash-manifest', action='store_true',
297 dest='youtube_include_dash_manifest', default=False,
298 help='Try to download the DASH manifest on YouTube videos (experimental)')
299
300 authentication.add_option('-u', '--username',
301 dest='username', metavar='USERNAME', help='account username')
302 authentication.add_option('-p', '--password',
303 dest='password', metavar='PASSWORD', help='account password')
304 authentication.add_option('-n', '--netrc',
305 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
306 authentication.add_option('--video-password',
307 dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
308
309
310 video_format.add_option('-f', '--format',
311 action='store', dest='format', metavar='FORMAT', default=None,
312 help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestaudio", "worst", and "worstaudio". By default, youtube-dl will pick the best quality.')
313 video_format.add_option('--all-formats',
314 action='store_const', dest='format', help='download all available video formats', const='all')
315 video_format.add_option('--prefer-free-formats',
316 action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
317 video_format.add_option('--max-quality',
318 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
319 video_format.add_option('-F', '--list-formats',
320 action='store_true', dest='listformats', help='list all available formats')
321
322 subtitles.add_option('--write-sub', '--write-srt',
323 action='store_true', dest='writesubtitles',
324 help='write subtitle file', default=False)
325 subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
326 action='store_true', dest='writeautomaticsub',
327 help='write automatic subtitle file (youtube only)', default=False)
328 subtitles.add_option('--all-subs',
329 action='store_true', dest='allsubtitles',
330 help='downloads all the available subtitles of the video', default=False)
331 subtitles.add_option('--list-subs',
332 action='store_true', dest='listsubtitles',
333 help='lists all available subtitles for the video', default=False)
334 subtitles.add_option('--sub-format',
335 action='store', dest='subtitlesformat', metavar='FORMAT',
336 help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
337 subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
338 action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
339 default=[], callback=_comma_separated_values_options_callback,
340 help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
341
342 downloader.add_option('-r', '--rate-limit',
343 dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
344 downloader.add_option('-R', '--retries',
345 dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
346 downloader.add_option('--buffer-size',
347 dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
348 downloader.add_option('--no-resize-buffer',
349 action='store_true', dest='noresizebuffer',
350 help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
351 downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
352
353 verbosity.add_option('-q', '--quiet',
354 action='store_true', dest='quiet', help='activates quiet mode', default=False)
355 verbosity.add_option('-s', '--simulate',
356 action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
357 verbosity.add_option('--skip-download',
358 action='store_true', dest='skip_download', help='do not download the video', default=False)
359 verbosity.add_option('-g', '--get-url',
360 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
361 verbosity.add_option('-e', '--get-title',
362 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
363 verbosity.add_option('--get-id',
364 action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
365 verbosity.add_option('--get-thumbnail',
366 action='store_true', dest='getthumbnail',
367 help='simulate, quiet but print thumbnail URL', default=False)
368 verbosity.add_option('--get-description',
369 action='store_true', dest='getdescription',
370 help='simulate, quiet but print video description', default=False)
371 verbosity.add_option('--get-duration',
372 action='store_true', dest='getduration',
373 help='simulate, quiet but print video length', default=False)
374 verbosity.add_option('--get-filename',
375 action='store_true', dest='getfilename',
376 help='simulate, quiet but print output filename', default=False)
377 verbosity.add_option('--get-format',
378 action='store_true', dest='getformat',
379 help='simulate, quiet but print output format', default=False)
380 verbosity.add_option('-j', '--dump-json',
381 action='store_true', dest='dumpjson',
382 help='simulate, quiet but print JSON information', default=False)
383 verbosity.add_option('--newline',
384 action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
385 verbosity.add_option('--no-progress',
386 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
387 verbosity.add_option('--console-title',
388 action='store_true', dest='consoletitle',
389 help='display progress in console titlebar', default=False)
390 verbosity.add_option('-v', '--verbose',
391 action='store_true', dest='verbose', help='print various debugging information', default=False)
392 verbosity.add_option('--dump-intermediate-pages',
393 action='store_true', dest='dump_intermediate_pages', default=False,
394 help='print downloaded pages to debug problems (very verbose)')
395 verbosity.add_option('--write-pages',
396 action='store_true', dest='write_pages', default=False,
397 help='Write downloaded intermediary pages to files in the current directory to debug problems')
398 verbosity.add_option('--youtube-print-sig-code',
399 action='store_true', dest='youtube_print_sig_code', default=False,
400 help=optparse.SUPPRESS_HELP)
401 verbosity.add_option('--print-traffic',
402 dest='debug_printtraffic', action='store_true', default=False,
403 help='Display sent and read HTTP traffic')
404
405
406 filesystem.add_option('-t', '--title',
407 action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
408 filesystem.add_option('--id',
409 action='store_true', dest='useid', help='use only video ID in file name', default=False)
410 filesystem.add_option('-l', '--literal',
411 action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
412 filesystem.add_option('-A', '--auto-number',
413 action='store_true', dest='autonumber',
414 help='number downloaded files starting from 00000', default=False)
415 filesystem.add_option('-o', '--output',
416 dest='outtmpl', metavar='TEMPLATE',
417 help=('output filename template. Use %(title)s to get the title, '
418 '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
419 '%(autonumber)s to get an automatically incremented number, '
420 '%(ext)s for the filename extension, '
421 '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
422 '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
423 '%(upload_date)s for the upload date (YYYYMMDD), '
424 '%(extractor)s for the provider (youtube, metacafe, etc), '
425 '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
426 '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
427 'Use - to output to stdout. Can also be used to download to a different directory, '
428 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
429 filesystem.add_option('--autonumber-size',
430 dest='autonumber_size', metavar='NUMBER',
431 help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
432 filesystem.add_option('--restrict-filenames',
433 action='store_true', dest='restrictfilenames',
434 help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
435 filesystem.add_option('-a', '--batch-file',
436 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
437 filesystem.add_option('--load-info',
438 dest='load_info_filename', metavar='FILE',
439 help='json file containing the video information (created with the "--write-json" option)')
440 filesystem.add_option('-w', '--no-overwrites',
441 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
442 filesystem.add_option('-c', '--continue',
443 action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
444 filesystem.add_option('--no-continue',
445 action='store_false', dest='continue_dl',
446 help='do not resume partially downloaded files (restart from beginning)')
447 filesystem.add_option('--cookies',
448 dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
449 filesystem.add_option('--no-part',
450 action='store_true', dest='nopart', help='do not use .part files', default=False)
451 filesystem.add_option('--no-mtime',
452 action='store_false', dest='updatetime',
453 help='do not use the Last-modified header to set the file modification time', default=True)
454 filesystem.add_option('--write-description',
455 action='store_true', dest='writedescription',
456 help='write video description to a .description file', default=False)
457 filesystem.add_option('--write-info-json',
458 action='store_true', dest='writeinfojson',
459 help='write video metadata to a .info.json file', default=False)
460 filesystem.add_option('--write-annotations',
461 action='store_true', dest='writeannotations',
462 help='write video annotations to a .annotation file', default=False)
463 filesystem.add_option('--write-thumbnail',
464 action='store_true', dest='writethumbnail',
465 help='write thumbnail image to disk', default=False)
466
467
468 postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
469 help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
470 postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
471 help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
472 postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
473 help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
474 postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
475 help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
476 postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
477 help='keeps the video file on disk after the post-processing; the video is erased by default')
478 postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
479 help='do not overwrite post-processed files; the post-processed files are overwritten by default')
480 postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
481 help='embed subtitles in the video (only for mp4 videos)')
482 postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
483 help='write metadata to the video file')
484 postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
485 help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
486 postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
487 help='Prefer avconv over ffmpeg for running the postprocessors (default)')
488 postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
489 help='Prefer ffmpeg over avconv for running the postprocessors')
490
491
492 parser.add_option_group(general)
493 parser.add_option_group(selection)
494 parser.add_option_group(downloader)
495 parser.add_option_group(filesystem)
496 parser.add_option_group(verbosity)
497 parser.add_option_group(video_format)
498 parser.add_option_group(subtitles)
499 parser.add_option_group(authentication)
500 parser.add_option_group(postproc)
501
502 if overrideArguments is not None:
503 opts, args = parser.parse_args(overrideArguments)
504 if opts.verbose:
505 write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
506 else:
507 commandLineConf = sys.argv[1:]
508 if '--ignore-config' in commandLineConf:
509 systemConf = []
510 userConf = []
511 else:
512 systemConf = _readOptions('/etc/youtube-dl.conf')
513 if '--ignore-config' in systemConf:
514 userConf = []
515 else:
516 userConf = _readUserConf()
517 argv = systemConf + userConf + commandLineConf
518
519 opts, args = parser.parse_args(argv)
520 if opts.verbose:
521 write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
522 write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
523 write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
524 write_string(u'[debug] Encodings: locale %r, fs %r, out %r, pref: %r\n' %
525 (locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, preferredencoding()))
526
527 return parser, opts, args
528
529
530 def _real_main(argv=None):
531 # Compatibility fixes for Windows
532 if sys.platform == 'win32':
533 # https://github.com/rg3/youtube-dl/issues/820
534 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
535
536 setproctitle(u'youtube-dl')
537
538 parser, opts, args = parseOpts(argv)
539
540 # Set user agent
541 if opts.user_agent is not None:
542 std_headers['User-Agent'] = opts.user_agent
543
544 # Set referer
545 if opts.referer is not None:
546 std_headers['Referer'] = opts.referer
547
548 # Dump user agent
549 if opts.dump_user_agent:
550 compat_print(std_headers['User-Agent'])
551 sys.exit(0)
552
553 # Batch file verification
554 batchurls = []
555 if opts.batchfile is not None:
556 try:
557 if opts.batchfile == '-':
558 batchfd = sys.stdin
559 else:
560 batchfd = open(opts.batchfile, 'r')
561 batchurls = batchfd.readlines()
562 batchurls = [x.strip() for x in batchurls]
563 batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
564 if opts.verbose:
565 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
566 except IOError:
567 sys.exit(u'ERROR: batch file could not be read')
568 all_urls = batchurls + args
569 all_urls = [url.strip() for url in all_urls]
570 _enc = preferredencoding()
571 all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
572
573 extractors = gen_extractors()
574
575 if opts.list_extractors:
576 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
577 compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
578 matchedUrls = [url for url in all_urls if ie.suitable(url)]
579 for mu in matchedUrls:
580 compat_print(u' ' + mu)
581 sys.exit(0)
582 if opts.list_extractor_descriptions:
583 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
584 if not ie._WORKING:
585 continue
586 desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
587 if desc is False:
588 continue
589 if hasattr(ie, 'SEARCH_KEY'):
590 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
591 _COUNTS = (u'', u'5', u'10', u'all')
592 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
593 compat_print(desc)
594 sys.exit(0)
595
596
597 # Conflicting, missing and erroneous options
598 if opts.usenetrc and (opts.username is not None or opts.password is not None):
599 parser.error(u'using .netrc conflicts with giving username/password')
600 if opts.password is not None and opts.username is None:
601 parser.error(u'account username missing\n')
602 if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
603 parser.error(u'using output template conflicts with using title, video ID or auto number')
604 if opts.usetitle and opts.useid:
605 parser.error(u'using title conflicts with using video ID')
606 if opts.username is not None and opts.password is None:
607 opts.password = getpass.getpass(u'Type account password and press return:')
608 if opts.ratelimit is not None:
609 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
610 if numeric_limit is None:
611 parser.error(u'invalid rate limit specified')
612 opts.ratelimit = numeric_limit
613 if opts.min_filesize is not None:
614 numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
615 if numeric_limit is None:
616 parser.error(u'invalid min_filesize specified')
617 opts.min_filesize = numeric_limit
618 if opts.max_filesize is not None:
619 numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
620 if numeric_limit is None:
621 parser.error(u'invalid max_filesize specified')
622 opts.max_filesize = numeric_limit
623 if opts.retries is not None:
624 try:
625 opts.retries = int(opts.retries)
626 except (TypeError, ValueError):
627 parser.error(u'invalid retry count specified')
628 if opts.buffersize is not None:
629 numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
630 if numeric_buffersize is None:
631 parser.error(u'invalid buffer size specified')
632 opts.buffersize = numeric_buffersize
633 if opts.playliststart <= 0:
634 raise ValueError(u'Playlist start must be positive')
635 if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
636 raise ValueError(u'Playlist end must be greater than playlist start')
637 if opts.extractaudio:
638 if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
639 parser.error(u'invalid audio format specified')
640 if opts.audioquality:
641 opts.audioquality = opts.audioquality.strip('k').strip('K')
642 if not opts.audioquality.isdigit():
643 parser.error(u'invalid audio quality specified')
644 if opts.recodevideo is not None:
645 if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
646 parser.error(u'invalid video recode format specified')
647 if opts.date is not None:
648 date = DateRange.day(opts.date)
649 else:
650 date = DateRange(opts.dateafter, opts.datebefore)
651 if opts.default_search not in ('auto', None) and ':' not in opts.default_search:
652 parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
653
654 # Do not download videos when there are audio-only formats
655 if opts.extractaudio and not opts.keepvideo and opts.format is None:
656 opts.format = 'bestaudio/best'
657
658 # --all-sub automatically sets --write-sub if --write-auto-sub is not given
659 # this was the old behaviour if only --all-sub was given.
660 if opts.allsubtitles and (opts.writeautomaticsub == False):
661 opts.writesubtitles = True
662
663 if sys.version_info < (3,):
664 # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
665 if opts.outtmpl is not None:
666 opts.outtmpl = opts.outtmpl.decode(preferredencoding())
667 outtmpl =((opts.outtmpl is not None and opts.outtmpl)
668 or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
669 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
670 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
671 or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
672 or (opts.useid and u'%(id)s.%(ext)s')
673 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
674 or u'%(title)s-%(id)s.%(ext)s')
675 if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
676 parser.error(u'Cannot download a video and extract audio into the same'
677 u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
678 u' template'.format(outtmpl))
679
680 any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
681 download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
682
683 ydl_opts = {
684 'usenetrc': opts.usenetrc,
685 'username': opts.username,
686 'password': opts.password,
687 'videopassword': opts.videopassword,
688 'quiet': (opts.quiet or any_printing),
689 'forceurl': opts.geturl,
690 'forcetitle': opts.gettitle,
691 'forceid': opts.getid,
692 'forcethumbnail': opts.getthumbnail,
693 'forcedescription': opts.getdescription,
694 'forceduration': opts.getduration,
695 'forcefilename': opts.getfilename,
696 'forceformat': opts.getformat,
697 'forcejson': opts.dumpjson,
698 'simulate': opts.simulate,
699 'skip_download': (opts.skip_download or opts.simulate or any_printing),
700 'format': opts.format,
701 'format_limit': opts.format_limit,
702 'listformats': opts.listformats,
703 'outtmpl': outtmpl,
704 'autonumber_size': opts.autonumber_size,
705 'restrictfilenames': opts.restrictfilenames,
706 'ignoreerrors': opts.ignoreerrors,
707 'ratelimit': opts.ratelimit,
708 'nooverwrites': opts.nooverwrites,
709 'retries': opts.retries,
710 'buffersize': opts.buffersize,
711 'noresizebuffer': opts.noresizebuffer,
712 'continuedl': opts.continue_dl,
713 'noprogress': opts.noprogress,
714 'progress_with_newline': opts.progress_with_newline,
715 'playliststart': opts.playliststart,
716 'playlistend': opts.playlistend,
717 'noplaylist': opts.noplaylist,
718 'logtostderr': opts.outtmpl == '-',
719 'consoletitle': opts.consoletitle,
720 'nopart': opts.nopart,
721 'updatetime': opts.updatetime,
722 'writedescription': opts.writedescription,
723 'writeannotations': opts.writeannotations,
724 'writeinfojson': opts.writeinfojson,
725 'writethumbnail': opts.writethumbnail,
726 'writesubtitles': opts.writesubtitles,
727 'writeautomaticsub': opts.writeautomaticsub,
728 'allsubtitles': opts.allsubtitles,
729 'listsubtitles': opts.listsubtitles,
730 'subtitlesformat': opts.subtitlesformat,
731 'subtitleslangs': opts.subtitleslangs,
732 'matchtitle': decodeOption(opts.matchtitle),
733 'rejecttitle': decodeOption(opts.rejecttitle),
734 'max_downloads': opts.max_downloads,
735 'prefer_free_formats': opts.prefer_free_formats,
736 'verbose': opts.verbose,
737 'dump_intermediate_pages': opts.dump_intermediate_pages,
738 'write_pages': opts.write_pages,
739 'test': opts.test,
740 'keepvideo': opts.keepvideo,
741 'min_filesize': opts.min_filesize,
742 'max_filesize': opts.max_filesize,
743 'min_views': opts.min_views,
744 'max_views': opts.max_views,
745 'daterange': date,
746 'cachedir': opts.cachedir,
747 'youtube_print_sig_code': opts.youtube_print_sig_code,
748 'age_limit': opts.age_limit,
749 'download_archive': download_archive_fn,
750 'cookiefile': opts.cookiefile,
751 'nocheckcertificate': opts.no_check_certificate,
752 'proxy': opts.proxy,
753 'socket_timeout': opts.socket_timeout,
754 'bidi_workaround': opts.bidi_workaround,
755 'debug_printtraffic': opts.debug_printtraffic,
756 'prefer_ffmpeg': opts.prefer_ffmpeg,
757 'include_ads': opts.include_ads,
758 'default_search': opts.default_search,
759 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
760 }
761
762 with YoutubeDL(ydl_opts) as ydl:
763 ydl.print_debug_header()
764 ydl.add_default_info_extractors()
765
766 # PostProcessors
767 # Add the metadata pp first, the other pps will copy it
768 if opts.addmetadata:
769 ydl.add_post_processor(FFmpegMetadataPP())
770 if opts.extractaudio:
771 ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
772 if opts.recodevideo:
773 ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
774 if opts.embedsubtitles:
775 ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
776 if opts.xattrs:
777 ydl.add_post_processor(XAttrMetadataPP())
778
779 # Update version
780 if opts.update_self:
781 update_self(ydl.to_screen, opts.verbose)
782
783 # Maybe do nothing
784 if (len(all_urls) < 1) and (opts.load_info_filename is None):
785 if not opts.update_self:
786 parser.error(u'you must provide at least one URL')
787 else:
788 sys.exit()
789
790 try:
791 if opts.load_info_filename is not None:
792 retcode = ydl.download_with_info_file(opts.load_info_filename)
793 else:
794 retcode = ydl.download(all_urls)
795 except MaxDownloadsReached:
796 ydl.to_screen(u'--max-download limit reached, aborting.')
797 retcode = 101
798
799 sys.exit(retcode)
800
801
802 def main(argv=None):
803 try:
804 _real_main(argv)
805 except DownloadError:
806 sys.exit(1)
807 except SameFileError:
808 sys.exit(u'ERROR: fixed output name but more than one file to download')
809 except KeyboardInterrupt:
810 sys.exit(u'\nERROR: Interrupted by user')