]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py
ace80f14b8bdac8dca7c3d87e08179ead17c5c02
[youtubedl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 from __future__ import absolute_import, unicode_literals
5
6 import collections
7 import contextlib
8 import copy
9 import datetime
10 import errno
11 import fileinput
12 import io
13 import itertools
14 import json
15 import locale
16 import operator
17 import os
18 import platform
19 import re
20 import shutil
21 import subprocess
22 import socket
23 import sys
24 import time
25 import tokenize
26 import traceback
27 import random
28
29 from string import ascii_letters
30
31 from .compat import (
32 compat_basestring,
33 compat_cookiejar,
34 compat_get_terminal_size,
35 compat_http_client,
36 compat_kwargs,
37 compat_numeric_types,
38 compat_os_name,
39 compat_str,
40 compat_tokenize_tokenize,
41 compat_urllib_error,
42 compat_urllib_request,
43 compat_urllib_request_DataHandler,
44 )
45 from .utils import (
46 age_restricted,
47 args_to_str,
48 ContentTooShortError,
49 date_from_str,
50 DateRange,
51 DEFAULT_OUTTMPL,
52 determine_ext,
53 determine_protocol,
54 DownloadError,
55 encode_compat_str,
56 encodeFilename,
57 error_to_compat_str,
58 expand_path,
59 ExtractorError,
60 format_bytes,
61 formatSeconds,
62 GeoRestrictedError,
63 int_or_none,
64 ISO3166Utils,
65 locked_file,
66 make_HTTPS_handler,
67 MaxDownloadsReached,
68 orderedSet,
69 PagedList,
70 parse_filesize,
71 PerRequestProxyHandler,
72 platform_name,
73 PostProcessingError,
74 preferredencoding,
75 prepend_extension,
76 register_socks_protocols,
77 render_table,
78 replace_extension,
79 SameFileError,
80 sanitize_filename,
81 sanitize_path,
82 sanitize_url,
83 sanitized_Request,
84 std_headers,
85 subtitles_filename,
86 UnavailableVideoError,
87 url_basename,
88 version_tuple,
89 write_json_file,
90 write_string,
91 YoutubeDLCookieProcessor,
92 YoutubeDLHandler,
93 )
94 from .cache import Cache
95 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
96 from .extractor.openload import PhantomJSwrapper
97 from .downloader import get_suitable_downloader
98 from .downloader.rtmp import rtmpdump_version
99 from .postprocessor import (
100 FFmpegFixupM3u8PP,
101 FFmpegFixupM4aPP,
102 FFmpegFixupStretchedPP,
103 FFmpegMergerPP,
104 FFmpegPostProcessor,
105 get_postprocessor,
106 )
107 from .version import __version__
108
109 if compat_os_name == 'nt':
110 import ctypes
111
112
113 class YoutubeDL(object):
114 """YoutubeDL class.
115
116 YoutubeDL objects are the ones responsible of downloading the
117 actual video file and writing it to disk if the user has requested
118 it, among some other tasks. In most cases there should be one per
119 program. As, given a video URL, the downloader doesn't know how to
120 extract all the needed information, task that InfoExtractors do, it
121 has to pass the URL to one of them.
122
123 For this, YoutubeDL objects have a method that allows
124 InfoExtractors to be registered in a given order. When it is passed
125 a URL, the YoutubeDL object handles it to the first InfoExtractor it
126 finds that reports being able to handle it. The InfoExtractor extracts
127 all the information about the video or videos the URL refers to, and
128 YoutubeDL process the extracted information, possibly using a File
129 Downloader to download the video.
130
131 YoutubeDL objects accept a lot of parameters. In order not to saturate
132 the object constructor with arguments, it receives a dictionary of
133 options instead. These options are available through the params
134 attribute for the InfoExtractors to use. The YoutubeDL also
135 registers itself as the downloader in charge for the InfoExtractors
136 that are added to it, so this is a "mutual registration".
137
138 Available options:
139
140 username: Username for authentication purposes.
141 password: Password for authentication purposes.
142 videopassword: Password for accessing a video.
143 ap_mso: Adobe Pass multiple-system operator identifier.
144 ap_username: Multiple-system operator account username.
145 ap_password: Multiple-system operator account password.
146 usenetrc: Use netrc for authentication instead.
147 verbose: Print additional info to stdout.
148 quiet: Do not print messages to stdout.
149 no_warnings: Do not print out anything for warnings.
150 forceurl: Force printing final URL.
151 forcetitle: Force printing title.
152 forceid: Force printing ID.
153 forcethumbnail: Force printing thumbnail URL.
154 forcedescription: Force printing description.
155 forcefilename: Force printing final filename.
156 forceduration: Force printing duration.
157 forcejson: Force printing info_dict as JSON.
158 dump_single_json: Force printing the info_dict of the whole playlist
159 (or video) as a single JSON line.
160 simulate: Do not download the video files.
161 format: Video format code. See options.py for more information.
162 outtmpl: Template for output names.
163 restrictfilenames: Do not allow "&" and spaces in file names
164 ignoreerrors: Do not stop on download errors.
165 force_generic_extractor: Force downloader to use the generic extractor
166 nooverwrites: Prevent overwriting files.
167 playliststart: Playlist item to start at.
168 playlistend: Playlist item to end at.
169 playlist_items: Specific indices of playlist to download.
170 playlistreverse: Download playlist items in reverse order.
171 playlistrandom: Download playlist items in random order.
172 matchtitle: Download only matching titles.
173 rejecttitle: Reject downloads for matching titles.
174 logger: Log messages to a logging.Logger instance.
175 logtostderr: Log messages to stderr instead of stdout.
176 writedescription: Write the video description to a .description file
177 writeinfojson: Write the video description to a .info.json file
178 writeannotations: Write the video annotations to a .annotations.xml file
179 writethumbnail: Write the thumbnail image to a file
180 write_all_thumbnails: Write all thumbnail formats to files
181 writesubtitles: Write the video subtitles to a file
182 writeautomaticsub: Write the automatically generated subtitles to a file
183 allsubtitles: Downloads all the subtitles of the video
184 (requires writesubtitles or writeautomaticsub)
185 listsubtitles: Lists all available subtitles for the video
186 subtitlesformat: The format code for subtitles
187 subtitleslangs: List of languages of the subtitles to download
188 keepvideo: Keep the video file after post-processing
189 daterange: A DateRange object, download only if the upload_date is in the range.
190 skip_download: Skip the actual download of the video file
191 cachedir: Location of the cache files in the filesystem.
192 False to disable filesystem cache.
193 noplaylist: Download single video instead of a playlist if in doubt.
194 age_limit: An integer representing the user's age in years.
195 Unsuitable videos for the given age are skipped.
196 min_views: An integer representing the minimum view count the video
197 must have in order to not be skipped.
198 Videos without view count information are always
199 downloaded. None for no limit.
200 max_views: An integer representing the maximum view count.
201 Videos that are more popular than that are not
202 downloaded.
203 Videos without view count information are always
204 downloaded. None for no limit.
205 download_archive: File name of a file where all downloads are recorded.
206 Videos already present in the file are not downloaded
207 again.
208 cookiefile: File name where cookies should be read from and dumped to.
209 nocheckcertificate:Do not verify SSL certificates
210 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
211 At the moment, this is only supported by YouTube.
212 proxy: URL of the proxy server to use
213 geo_verification_proxy: URL of the proxy to use for IP address verification
214 on geo-restricted sites. (Experimental)
215 socket_timeout: Time to wait for unresponsive hosts, in seconds
216 bidi_workaround: Work around buggy terminals without bidirectional text
217 support, using fridibi
218 debug_printtraffic:Print out sent and received HTTP traffic
219 include_ads: Download ads as well
220 default_search: Prepend this string if an input url is not valid.
221 'auto' for elaborate guessing
222 encoding: Use this encoding instead of the system-specified.
223 extract_flat: Do not resolve URLs, return the immediate result.
224 Pass in 'in_playlist' to only show this behavior for
225 playlist items.
226 postprocessors: A list of dictionaries, each with an entry
227 * key: The name of the postprocessor. See
228 youtube_dl/postprocessor/__init__.py for a list.
229 as well as any further keyword arguments for the
230 postprocessor.
231 progress_hooks: A list of functions that get called on download
232 progress, with a dictionary with the entries
233 * status: One of "downloading", "error", or "finished".
234 Check this first and ignore unknown values.
235
236 If status is one of "downloading", or "finished", the
237 following properties may also be present:
238 * filename: The final filename (always present)
239 * tmpfilename: The filename we're currently writing to
240 * downloaded_bytes: Bytes on disk
241 * total_bytes: Size of the whole file, None if unknown
242 * total_bytes_estimate: Guess of the eventual file size,
243 None if unavailable.
244 * elapsed: The number of seconds since download started.
245 * eta: The estimated time in seconds, None if unknown
246 * speed: The download speed in bytes/second, None if
247 unknown
248 * fragment_index: The counter of the currently
249 downloaded video fragment.
250 * fragment_count: The number of fragments (= individual
251 files that will be merged)
252
253 Progress hooks are guaranteed to be called at least once
254 (with status "finished") if the download is successful.
255 merge_output_format: Extension to use when merging formats.
256 fixup: Automatically correct known faults of the file.
257 One of:
258 - "never": do nothing
259 - "warn": only emit a warning
260 - "detect_or_warn": check whether we can do anything
261 about it, warn otherwise (default)
262 source_address: (Experimental) Client-side IP address to bind to.
263 call_home: Boolean, true iff we are allowed to contact the
264 youtube-dl servers for debugging.
265 sleep_interval: Number of seconds to sleep before each download when
266 used alone or a lower bound of a range for randomized
267 sleep before each download (minimum possible number
268 of seconds to sleep) when used along with
269 max_sleep_interval.
270 max_sleep_interval:Upper bound of a range for randomized sleep before each
271 download (maximum possible number of seconds to sleep).
272 Must only be used along with sleep_interval.
273 Actual sleep time will be a random float from range
274 [sleep_interval; max_sleep_interval].
275 listformats: Print an overview of available video formats and exit.
276 list_thumbnails: Print a table of all thumbnails and exit.
277 match_filter: A function that gets called with the info_dict of
278 every video.
279 If it returns a message, the video is ignored.
280 If it returns None, the video is downloaded.
281 match_filter_func in utils.py is one example for this.
282 no_color: Do not emit color codes in output.
283 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
284 HTTP header (experimental)
285 geo_bypass_country:
286 Two-letter ISO 3166-2 country code that will be used for
287 explicit geographic restriction bypassing via faking
288 X-Forwarded-For HTTP header (experimental)
289
290 The following options determine which downloader is picked:
291 external_downloader: Executable of the external downloader to call.
292 None or unset for standard (built-in) downloader.
293 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
294 if True, otherwise use ffmpeg/avconv if False, otherwise
295 use downloader suggested by extractor if None.
296
297 The following parameters are not used by YoutubeDL itself, they are used by
298 the downloader (see youtube_dl/downloader/common.py):
299 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
300 noresizebuffer, retries, continuedl, noprogress, consoletitle,
301 xattr_set_filesize, external_downloader_args, hls_use_mpegts.
302
303 The following options are used by the post processors:
304 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
305 otherwise prefer avconv.
306 postprocessor_args: A list of additional command-line arguments for the
307 postprocessor.
308
309 The following options are used by the Youtube extractor:
310 youtube_include_dash_manifest: If True (default), DASH manifests and related
311 data will be downloaded and processed by extractor.
312 You can reduce network I/O by disabling it if you don't
313 care about DASH.
314 """
315
316 _NUMERIC_FIELDS = set((
317 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
318 'timestamp', 'upload_year', 'upload_month', 'upload_day',
319 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
320 'average_rating', 'comment_count', 'age_limit',
321 'start_time', 'end_time',
322 'chapter_number', 'season_number', 'episode_number',
323 'track_number', 'disc_number', 'release_year',
324 'playlist_index',
325 ))
326
327 params = None
328 _ies = []
329 _pps = []
330 _download_retcode = None
331 _num_downloads = None
332 _screen_file = None
333
334 def __init__(self, params=None, auto_init=True):
335 """Create a FileDownloader object with the given options."""
336 if params is None:
337 params = {}
338 self._ies = []
339 self._ies_instances = {}
340 self._pps = []
341 self._progress_hooks = []
342 self._download_retcode = 0
343 self._num_downloads = 0
344 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
345 self._err_file = sys.stderr
346 self.params = {
347 # Default parameters
348 'nocheckcertificate': False,
349 }
350 self.params.update(params)
351 self.cache = Cache(self)
352
353 def check_deprecated(param, option, suggestion):
354 if self.params.get(param) is not None:
355 self.report_warning(
356 '%s is deprecated. Use %s instead.' % (option, suggestion))
357 return True
358 return False
359
360 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
361 if self.params.get('geo_verification_proxy') is None:
362 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
363
364 check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
365 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
366 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
367
368 if params.get('bidi_workaround', False):
369 try:
370 import pty
371 master, slave = pty.openpty()
372 width = compat_get_terminal_size().columns
373 if width is None:
374 width_args = []
375 else:
376 width_args = ['-w', str(width)]
377 sp_kwargs = dict(
378 stdin=subprocess.PIPE,
379 stdout=slave,
380 stderr=self._err_file)
381 try:
382 self._output_process = subprocess.Popen(
383 ['bidiv'] + width_args, **sp_kwargs
384 )
385 except OSError:
386 self._output_process = subprocess.Popen(
387 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
388 self._output_channel = os.fdopen(master, 'rb')
389 except OSError as ose:
390 if ose.errno == errno.ENOENT:
391 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
392 else:
393 raise
394
395 if (sys.platform != 'win32' and
396 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
397 not params.get('restrictfilenames', False)):
398 # Unicode filesystem API will throw errors (#1474, #13027)
399 self.report_warning(
400 'Assuming --restrict-filenames since file system encoding '
401 'cannot encode all characters. '
402 'Set the LC_ALL environment variable to fix this.')
403 self.params['restrictfilenames'] = True
404
405 if isinstance(params.get('outtmpl'), bytes):
406 self.report_warning(
407 'Parameter outtmpl is bytes, but should be a unicode string. '
408 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
409
410 self._setup_opener()
411
412 if auto_init:
413 self.print_debug_header()
414 self.add_default_info_extractors()
415
416 for pp_def_raw in self.params.get('postprocessors', []):
417 pp_class = get_postprocessor(pp_def_raw['key'])
418 pp_def = dict(pp_def_raw)
419 del pp_def['key']
420 pp = pp_class(self, **compat_kwargs(pp_def))
421 self.add_post_processor(pp)
422
423 for ph in self.params.get('progress_hooks', []):
424 self.add_progress_hook(ph)
425
426 register_socks_protocols()
427
428 def warn_if_short_id(self, argv):
429 # short YouTube ID starting with dash?
430 idxs = [
431 i for i, a in enumerate(argv)
432 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
433 if idxs:
434 correct_argv = (
435 ['youtube-dl'] +
436 [a for i, a in enumerate(argv) if i not in idxs] +
437 ['--'] + [argv[i] for i in idxs]
438 )
439 self.report_warning(
440 'Long argument string detected. '
441 'Use -- to separate parameters and URLs, like this:\n%s\n' %
442 args_to_str(correct_argv))
443
444 def add_info_extractor(self, ie):
445 """Add an InfoExtractor object to the end of the list."""
446 self._ies.append(ie)
447 if not isinstance(ie, type):
448 self._ies_instances[ie.ie_key()] = ie
449 ie.set_downloader(self)
450
451 def get_info_extractor(self, ie_key):
452 """
453 Get an instance of an IE with name ie_key, it will try to get one from
454 the _ies list, if there's no instance it will create a new one and add
455 it to the extractor list.
456 """
457 ie = self._ies_instances.get(ie_key)
458 if ie is None:
459 ie = get_info_extractor(ie_key)()
460 self.add_info_extractor(ie)
461 return ie
462
463 def add_default_info_extractors(self):
464 """
465 Add the InfoExtractors returned by gen_extractors to the end of the list
466 """
467 for ie in gen_extractor_classes():
468 self.add_info_extractor(ie)
469
470 def add_post_processor(self, pp):
471 """Add a PostProcessor object to the end of the chain."""
472 self._pps.append(pp)
473 pp.set_downloader(self)
474
475 def add_progress_hook(self, ph):
476 """Add the progress hook (currently only for the file downloader)"""
477 self._progress_hooks.append(ph)
478
479 def _bidi_workaround(self, message):
480 if not hasattr(self, '_output_channel'):
481 return message
482
483 assert hasattr(self, '_output_process')
484 assert isinstance(message, compat_str)
485 line_count = message.count('\n') + 1
486 self._output_process.stdin.write((message + '\n').encode('utf-8'))
487 self._output_process.stdin.flush()
488 res = ''.join(self._output_channel.readline().decode('utf-8')
489 for _ in range(line_count))
490 return res[:-len('\n')]
491
492 def to_screen(self, message, skip_eol=False):
493 """Print message to stdout if not in quiet mode."""
494 return self.to_stdout(message, skip_eol, check_quiet=True)
495
496 def _write_string(self, s, out=None):
497 write_string(s, out=out, encoding=self.params.get('encoding'))
498
499 def to_stdout(self, message, skip_eol=False, check_quiet=False):
500 """Print message to stdout if not in quiet mode."""
501 if self.params.get('logger'):
502 self.params['logger'].debug(message)
503 elif not check_quiet or not self.params.get('quiet', False):
504 message = self._bidi_workaround(message)
505 terminator = ['\n', ''][skip_eol]
506 output = message + terminator
507
508 self._write_string(output, self._screen_file)
509
510 def to_stderr(self, message):
511 """Print message to stderr."""
512 assert isinstance(message, compat_str)
513 if self.params.get('logger'):
514 self.params['logger'].error(message)
515 else:
516 message = self._bidi_workaround(message)
517 output = message + '\n'
518 self._write_string(output, self._err_file)
519
520 def to_console_title(self, message):
521 if not self.params.get('consoletitle', False):
522 return
523 if compat_os_name == 'nt':
524 if ctypes.windll.kernel32.GetConsoleWindow():
525 # c_wchar_p() might not be necessary if `message` is
526 # already of type unicode()
527 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
528 elif 'TERM' in os.environ:
529 self._write_string('\033]0;%s\007' % message, self._screen_file)
530
531 def save_console_title(self):
532 if not self.params.get('consoletitle', False):
533 return
534 if compat_os_name != 'nt' and 'TERM' in os.environ:
535 # Save the title on stack
536 self._write_string('\033[22;0t', self._screen_file)
537
538 def restore_console_title(self):
539 if not self.params.get('consoletitle', False):
540 return
541 if compat_os_name != 'nt' and 'TERM' in os.environ:
542 # Restore the title from stack
543 self._write_string('\033[23;0t', self._screen_file)
544
545 def __enter__(self):
546 self.save_console_title()
547 return self
548
549 def __exit__(self, *args):
550 self.restore_console_title()
551
552 if self.params.get('cookiefile') is not None:
553 self.cookiejar.save()
554
555 def trouble(self, message=None, tb=None):
556 """Determine action to take when a download problem appears.
557
558 Depending on if the downloader has been configured to ignore
559 download errors or not, this method may throw an exception or
560 not when errors are found, after printing the message.
561
562 tb, if given, is additional traceback information.
563 """
564 if message is not None:
565 self.to_stderr(message)
566 if self.params.get('verbose'):
567 if tb is None:
568 if sys.exc_info()[0]: # if .trouble has been called from an except block
569 tb = ''
570 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
571 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
572 tb += encode_compat_str(traceback.format_exc())
573 else:
574 tb_data = traceback.format_list(traceback.extract_stack())
575 tb = ''.join(tb_data)
576 self.to_stderr(tb)
577 if not self.params.get('ignoreerrors', False):
578 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
579 exc_info = sys.exc_info()[1].exc_info
580 else:
581 exc_info = sys.exc_info()
582 raise DownloadError(message, exc_info)
583 self._download_retcode = 1
584
585 def report_warning(self, message):
586 '''
587 Print the message to stderr, it will be prefixed with 'WARNING:'
588 If stderr is a tty file the 'WARNING:' will be colored
589 '''
590 if self.params.get('logger') is not None:
591 self.params['logger'].warning(message)
592 else:
593 if self.params.get('no_warnings'):
594 return
595 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
596 _msg_header = '\033[0;33mWARNING:\033[0m'
597 else:
598 _msg_header = 'WARNING:'
599 warning_message = '%s %s' % (_msg_header, message)
600 self.to_stderr(warning_message)
601
602 def report_error(self, message, tb=None):
603 '''
604 Do the same as trouble, but prefixes the message with 'ERROR:', colored
605 in red if stderr is a tty file.
606 '''
607 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
608 _msg_header = '\033[0;31mERROR:\033[0m'
609 else:
610 _msg_header = 'ERROR:'
611 error_message = '%s %s' % (_msg_header, message)
612 self.trouble(error_message, tb)
613
614 def report_file_already_downloaded(self, file_name):
615 """Report file has already been fully downloaded."""
616 try:
617 self.to_screen('[download] %s has already been downloaded' % file_name)
618 except UnicodeEncodeError:
619 self.to_screen('[download] The file has already been downloaded')
620
621 def prepare_filename(self, info_dict):
622 """Generate the output filename."""
623 try:
624 template_dict = dict(info_dict)
625
626 template_dict['epoch'] = int(time.time())
627 autonumber_size = self.params.get('autonumber_size')
628 if autonumber_size is None:
629 autonumber_size = 5
630 template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
631 if template_dict.get('resolution') is None:
632 if template_dict.get('width') and template_dict.get('height'):
633 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
634 elif template_dict.get('height'):
635 template_dict['resolution'] = '%sp' % template_dict['height']
636 elif template_dict.get('width'):
637 template_dict['resolution'] = '%dx?' % template_dict['width']
638
639 sanitize = lambda k, v: sanitize_filename(
640 compat_str(v),
641 restricted=self.params.get('restrictfilenames'),
642 is_id=(k == 'id' or k.endswith('_id')))
643 template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
644 for k, v in template_dict.items()
645 if v is not None and not isinstance(v, (list, tuple, dict)))
646 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
647
648 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
649
650 # For fields playlist_index and autonumber convert all occurrences
651 # of %(field)s to %(field)0Nd for backward compatibility
652 field_size_compat_map = {
653 'playlist_index': len(str(template_dict['n_entries'])),
654 'autonumber': autonumber_size,
655 }
656 FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
657 mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
658 if mobj:
659 outtmpl = re.sub(
660 FIELD_SIZE_COMPAT_RE,
661 r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
662 outtmpl)
663
664 # Missing numeric fields used together with integer presentation types
665 # in format specification will break the argument substitution since
666 # string 'NA' is returned for missing fields. We will patch output
667 # template for missing fields to meet string presentation type.
668 for numeric_field in self._NUMERIC_FIELDS:
669 if numeric_field not in template_dict:
670 # As of [1] format syntax is:
671 # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
672 # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
673 FORMAT_RE = r'''(?x)
674 (?<!%)
675 %
676 \({0}\) # mapping key
677 (?:[#0\-+ ]+)? # conversion flags (optional)
678 (?:\d+)? # minimum field width (optional)
679 (?:\.\d+)? # precision (optional)
680 [hlL]? # length modifier (optional)
681 [diouxXeEfFgGcrs%] # conversion type
682 '''
683 outtmpl = re.sub(
684 FORMAT_RE.format(numeric_field),
685 r'%({0})s'.format(numeric_field), outtmpl)
686
687 # expand_path translates '%%' into '%' and '$$' into '$'
688 # correspondingly that is not what we want since we need to keep
689 # '%%' intact for template dict substitution step. Working around
690 # with boundary-alike separator hack.
691 sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
692 outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
693
694 # outtmpl should be expand_path'ed before template dict substitution
695 # because meta fields may contain env variables we don't want to
696 # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
697 # title "Hello $PATH", we don't want `$PATH` to be expanded.
698 filename = expand_path(outtmpl).replace(sep, '') % template_dict
699
700 # Temporary fix for #4787
701 # 'Treat' all problem characters by passing filename through preferredencoding
702 # to workaround encoding issues with subprocess on python2 @ Windows
703 if sys.version_info < (3, 0) and sys.platform == 'win32':
704 filename = encodeFilename(filename, True).decode(preferredencoding())
705 return sanitize_path(filename)
706 except ValueError as err:
707 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
708 return None
709
710 def _match_entry(self, info_dict, incomplete):
711 """ Returns None iff the file should be downloaded """
712
713 video_title = info_dict.get('title', info_dict.get('id', 'video'))
714 if 'title' in info_dict:
715 # This can happen when we're just evaluating the playlist
716 title = info_dict['title']
717 matchtitle = self.params.get('matchtitle', False)
718 if matchtitle:
719 if not re.search(matchtitle, title, re.IGNORECASE):
720 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
721 rejecttitle = self.params.get('rejecttitle', False)
722 if rejecttitle:
723 if re.search(rejecttitle, title, re.IGNORECASE):
724 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
725 date = info_dict.get('upload_date')
726 if date is not None:
727 dateRange = self.params.get('daterange', DateRange())
728 if date not in dateRange:
729 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
730 view_count = info_dict.get('view_count')
731 if view_count is not None:
732 min_views = self.params.get('min_views')
733 if min_views is not None and view_count < min_views:
734 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
735 max_views = self.params.get('max_views')
736 if max_views is not None and view_count > max_views:
737 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
738 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
739 return 'Skipping "%s" because it is age restricted' % video_title
740 if self.in_download_archive(info_dict):
741 return '%s has already been recorded in archive' % video_title
742
743 if not incomplete:
744 match_filter = self.params.get('match_filter')
745 if match_filter is not None:
746 ret = match_filter(info_dict)
747 if ret is not None:
748 return ret
749
750 return None
751
752 @staticmethod
753 def add_extra_info(info_dict, extra_info):
754 '''Set the keys from extra_info in info dict if they are missing'''
755 for key, value in extra_info.items():
756 info_dict.setdefault(key, value)
757
758 def extract_info(self, url, download=True, ie_key=None, extra_info={},
759 process=True, force_generic_extractor=False):
760 '''
761 Returns a list with a dictionary for each video we find.
762 If 'download', also downloads the videos.
763 extra_info is a dict containing the extra values to add to each result
764 '''
765
766 if not ie_key and force_generic_extractor:
767 ie_key = 'Generic'
768
769 if ie_key:
770 ies = [self.get_info_extractor(ie_key)]
771 else:
772 ies = self._ies
773
774 for ie in ies:
775 if not ie.suitable(url):
776 continue
777
778 ie = self.get_info_extractor(ie.ie_key())
779 if not ie.working():
780 self.report_warning('The program functionality for this site has been marked as broken, '
781 'and will probably not work.')
782
783 try:
784 ie_result = ie.extract(url)
785 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
786 break
787 if isinstance(ie_result, list):
788 # Backwards compatibility: old IE result format
789 ie_result = {
790 '_type': 'compat_list',
791 'entries': ie_result,
792 }
793 self.add_default_extra_info(ie_result, ie, url)
794 if process:
795 return self.process_ie_result(ie_result, download, extra_info)
796 else:
797 return ie_result
798 except GeoRestrictedError as e:
799 msg = e.msg
800 if e.countries:
801 msg += '\nThis video is available in %s.' % ', '.join(
802 map(ISO3166Utils.short2full, e.countries))
803 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
804 self.report_error(msg)
805 break
806 except ExtractorError as e: # An error we somewhat expected
807 self.report_error(compat_str(e), e.format_traceback())
808 break
809 except MaxDownloadsReached:
810 raise
811 except Exception as e:
812 if self.params.get('ignoreerrors', False):
813 self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
814 break
815 else:
816 raise
817 else:
818 self.report_error('no suitable InfoExtractor for URL %s' % url)
819
820 def add_default_extra_info(self, ie_result, ie, url):
821 self.add_extra_info(ie_result, {
822 'extractor': ie.IE_NAME,
823 'webpage_url': url,
824 'webpage_url_basename': url_basename(url),
825 'extractor_key': ie.ie_key(),
826 })
827
828 def process_ie_result(self, ie_result, download=True, extra_info={}):
829 """
830 Take the result of the ie(may be modified) and resolve all unresolved
831 references (URLs, playlist items).
832
833 It will also download the videos if 'download'.
834 Returns the resolved ie_result.
835 """
836 result_type = ie_result.get('_type', 'video')
837
838 if result_type in ('url', 'url_transparent'):
839 ie_result['url'] = sanitize_url(ie_result['url'])
840 extract_flat = self.params.get('extract_flat', False)
841 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
842 extract_flat is True):
843 if self.params.get('forcejson', False):
844 self.to_stdout(json.dumps(ie_result))
845 return ie_result
846
847 if result_type == 'video':
848 self.add_extra_info(ie_result, extra_info)
849 return self.process_video_result(ie_result, download=download)
850 elif result_type == 'url':
851 # We have to add extra_info to the results because it may be
852 # contained in a playlist
853 return self.extract_info(ie_result['url'],
854 download,
855 ie_key=ie_result.get('ie_key'),
856 extra_info=extra_info)
857 elif result_type == 'url_transparent':
858 # Use the information from the embedding page
859 info = self.extract_info(
860 ie_result['url'], ie_key=ie_result.get('ie_key'),
861 extra_info=extra_info, download=False, process=False)
862
863 # extract_info may return None when ignoreerrors is enabled and
864 # extraction failed with an error, don't crash and return early
865 # in this case
866 if not info:
867 return info
868
869 force_properties = dict(
870 (k, v) for k, v in ie_result.items() if v is not None)
871 for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
872 if f in force_properties:
873 del force_properties[f]
874 new_result = info.copy()
875 new_result.update(force_properties)
876
877 # Extracted info may not be a video result (i.e.
878 # info.get('_type', 'video') != video) but rather an url or
879 # url_transparent. In such cases outer metadata (from ie_result)
880 # should be propagated to inner one (info). For this to happen
881 # _type of info should be overridden with url_transparent. This
882 # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
883 if new_result.get('_type') == 'url':
884 new_result['_type'] = 'url_transparent'
885
886 return self.process_ie_result(
887 new_result, download=download, extra_info=extra_info)
888 elif result_type in ('playlist', 'multi_video'):
889 # We process each entry in the playlist
890 playlist = ie_result.get('title') or ie_result.get('id')
891 self.to_screen('[download] Downloading playlist: %s' % playlist)
892
893 playlist_results = []
894
895 playliststart = self.params.get('playliststart', 1) - 1
896 playlistend = self.params.get('playlistend')
897 # For backwards compatibility, interpret -1 as whole list
898 if playlistend == -1:
899 playlistend = None
900
901 playlistitems_str = self.params.get('playlist_items')
902 playlistitems = None
903 if playlistitems_str is not None:
904 def iter_playlistitems(format):
905 for string_segment in format.split(','):
906 if '-' in string_segment:
907 start, end = string_segment.split('-')
908 for item in range(int(start), int(end) + 1):
909 yield int(item)
910 else:
911 yield int(string_segment)
912 playlistitems = orderedSet(iter_playlistitems(playlistitems_str))
913
914 ie_entries = ie_result['entries']
915
916 def make_playlistitems_entries(list_ie_entries):
917 num_entries = len(list_ie_entries)
918 return [
919 list_ie_entries[i - 1] for i in playlistitems
920 if -num_entries <= i - 1 < num_entries]
921
922 def report_download(num_entries):
923 self.to_screen(
924 '[%s] playlist %s: Downloading %d videos' %
925 (ie_result['extractor'], playlist, num_entries))
926
927 if isinstance(ie_entries, list):
928 n_all_entries = len(ie_entries)
929 if playlistitems:
930 entries = make_playlistitems_entries(ie_entries)
931 else:
932 entries = ie_entries[playliststart:playlistend]
933 n_entries = len(entries)
934 self.to_screen(
935 '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
936 (ie_result['extractor'], playlist, n_all_entries, n_entries))
937 elif isinstance(ie_entries, PagedList):
938 if playlistitems:
939 entries = []
940 for item in playlistitems:
941 entries.extend(ie_entries.getslice(
942 item - 1, item
943 ))
944 else:
945 entries = ie_entries.getslice(
946 playliststart, playlistend)
947 n_entries = len(entries)
948 report_download(n_entries)
949 else: # iterable
950 if playlistitems:
951 entries = make_playlistitems_entries(list(itertools.islice(
952 ie_entries, 0, max(playlistitems))))
953 else:
954 entries = list(itertools.islice(
955 ie_entries, playliststart, playlistend))
956 n_entries = len(entries)
957 report_download(n_entries)
958
959 if self.params.get('playlistreverse', False):
960 entries = entries[::-1]
961
962 if self.params.get('playlistrandom', False):
963 random.shuffle(entries)
964
965 x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
966
967 for i, entry in enumerate(entries, 1):
968 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
969 # This __x_forwarded_for_ip thing is a bit ugly but requires
970 # minimal changes
971 if x_forwarded_for:
972 entry['__x_forwarded_for_ip'] = x_forwarded_for
973 extra = {
974 'n_entries': n_entries,
975 'playlist': playlist,
976 'playlist_id': ie_result.get('id'),
977 'playlist_title': ie_result.get('title'),
978 'playlist_uploader': ie_result.get('uploader'),
979 'playlist_uploader_id': ie_result.get('uploader_id'),
980 'playlist_index': i + playliststart,
981 'extractor': ie_result['extractor'],
982 'webpage_url': ie_result['webpage_url'],
983 'webpage_url_basename': url_basename(ie_result['webpage_url']),
984 'extractor_key': ie_result['extractor_key'],
985 }
986
987 reason = self._match_entry(entry, incomplete=True)
988 if reason is not None:
989 self.to_screen('[download] ' + reason)
990 continue
991
992 entry_result = self.process_ie_result(entry,
993 download=download,
994 extra_info=extra)
995 playlist_results.append(entry_result)
996 ie_result['entries'] = playlist_results
997 self.to_screen('[download] Finished downloading playlist: %s' % playlist)
998 return ie_result
999 elif result_type == 'compat_list':
1000 self.report_warning(
1001 'Extractor %s returned a compat_list result. '
1002 'It needs to be updated.' % ie_result.get('extractor'))
1003
1004 def _fixup(r):
1005 self.add_extra_info(
1006 r,
1007 {
1008 'extractor': ie_result['extractor'],
1009 'webpage_url': ie_result['webpage_url'],
1010 'webpage_url_basename': url_basename(ie_result['webpage_url']),
1011 'extractor_key': ie_result['extractor_key'],
1012 }
1013 )
1014 return r
1015 ie_result['entries'] = [
1016 self.process_ie_result(_fixup(r), download, extra_info)
1017 for r in ie_result['entries']
1018 ]
1019 return ie_result
1020 else:
1021 raise Exception('Invalid result type: %s' % result_type)
1022
1023 def _build_format_filter(self, filter_spec):
1024 " Returns a function to filter the formats according to the filter_spec "
1025
1026 OPERATORS = {
1027 '<': operator.lt,
1028 '<=': operator.le,
1029 '>': operator.gt,
1030 '>=': operator.ge,
1031 '=': operator.eq,
1032 '!=': operator.ne,
1033 }
1034 operator_rex = re.compile(r'''(?x)\s*
1035 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
1036 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1037 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1038 $
1039 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
1040 m = operator_rex.search(filter_spec)
1041 if m:
1042 try:
1043 comparison_value = int(m.group('value'))
1044 except ValueError:
1045 comparison_value = parse_filesize(m.group('value'))
1046 if comparison_value is None:
1047 comparison_value = parse_filesize(m.group('value') + 'B')
1048 if comparison_value is None:
1049 raise ValueError(
1050 'Invalid value %r in format specification %r' % (
1051 m.group('value'), filter_spec))
1052 op = OPERATORS[m.group('op')]
1053
1054 if not m:
1055 STR_OPERATORS = {
1056 '=': operator.eq,
1057 '!=': operator.ne,
1058 '^=': lambda attr, value: attr.startswith(value),
1059 '$=': lambda attr, value: attr.endswith(value),
1060 '*=': lambda attr, value: value in attr,
1061 }
1062 str_operator_rex = re.compile(r'''(?x)
1063 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
1064 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
1065 \s*(?P<value>[a-zA-Z0-9._-]+)
1066 \s*$
1067 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
1068 m = str_operator_rex.search(filter_spec)
1069 if m:
1070 comparison_value = m.group('value')
1071 op = STR_OPERATORS[m.group('op')]
1072
1073 if not m:
1074 raise ValueError('Invalid filter specification %r' % filter_spec)
1075
1076 def _filter(f):
1077 actual_value = f.get(m.group('key'))
1078 if actual_value is None:
1079 return m.group('none_inclusive')
1080 return op(actual_value, comparison_value)
1081 return _filter
1082
1083 def _default_format_spec(self, info_dict, download=True):
1084
1085 def can_merge():
1086 merger = FFmpegMergerPP(self)
1087 return merger.available and merger.can_merge()
1088
1089 def prefer_best():
1090 if self.params.get('simulate', False):
1091 return False
1092 if not download:
1093 return False
1094 if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
1095 return True
1096 if info_dict.get('is_live'):
1097 return True
1098 if not can_merge():
1099 return True
1100 return False
1101
1102 req_format_list = ['bestvideo+bestaudio', 'best']
1103 if prefer_best():
1104 req_format_list.reverse()
1105 return '/'.join(req_format_list)
1106
1107 def build_format_selector(self, format_spec):
1108 def syntax_error(note, start):
1109 message = (
1110 'Invalid format specification: '
1111 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
1112 return SyntaxError(message)
1113
1114 PICKFIRST = 'PICKFIRST'
1115 MERGE = 'MERGE'
1116 SINGLE = 'SINGLE'
1117 GROUP = 'GROUP'
1118 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1119
1120 def _parse_filter(tokens):
1121 filter_parts = []
1122 for type, string, start, _, _ in tokens:
1123 if type == tokenize.OP and string == ']':
1124 return ''.join(filter_parts)
1125 else:
1126 filter_parts.append(string)
1127
1128 def _remove_unused_ops(tokens):
1129 # Remove operators that we don't use and join them with the surrounding strings
1130 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1131 ALLOWED_OPS = ('/', '+', ',', '(', ')')
1132 last_string, last_start, last_end, last_line = None, None, None, None
1133 for type, string, start, end, line in tokens:
1134 if type == tokenize.OP and string == '[':
1135 if last_string:
1136 yield tokenize.NAME, last_string, last_start, last_end, last_line
1137 last_string = None
1138 yield type, string, start, end, line
1139 # everything inside brackets will be handled by _parse_filter
1140 for type, string, start, end, line in tokens:
1141 yield type, string, start, end, line
1142 if type == tokenize.OP and string == ']':
1143 break
1144 elif type == tokenize.OP and string in ALLOWED_OPS:
1145 if last_string:
1146 yield tokenize.NAME, last_string, last_start, last_end, last_line
1147 last_string = None
1148 yield type, string, start, end, line
1149 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1150 if not last_string:
1151 last_string = string
1152 last_start = start
1153 last_end = end
1154 else:
1155 last_string += string
1156 if last_string:
1157 yield tokenize.NAME, last_string, last_start, last_end, last_line
1158
1159 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1160 selectors = []
1161 current_selector = None
1162 for type, string, start, _, _ in tokens:
1163 # ENCODING is only defined in python 3.x
1164 if type == getattr(tokenize, 'ENCODING', None):
1165 continue
1166 elif type in [tokenize.NAME, tokenize.NUMBER]:
1167 current_selector = FormatSelector(SINGLE, string, [])
1168 elif type == tokenize.OP:
1169 if string == ')':
1170 if not inside_group:
1171 # ')' will be handled by the parentheses group
1172 tokens.restore_last_token()
1173 break
1174 elif inside_merge and string in ['/', ',']:
1175 tokens.restore_last_token()
1176 break
1177 elif inside_choice and string == ',':
1178 tokens.restore_last_token()
1179 break
1180 elif string == ',':
1181 if not current_selector:
1182 raise syntax_error('"," must follow a format selector', start)
1183 selectors.append(current_selector)
1184 current_selector = None
1185 elif string == '/':
1186 if not current_selector:
1187 raise syntax_error('"/" must follow a format selector', start)
1188 first_choice = current_selector
1189 second_choice = _parse_format_selection(tokens, inside_choice=True)
1190 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1191 elif string == '[':
1192 if not current_selector:
1193 current_selector = FormatSelector(SINGLE, 'best', [])
1194 format_filter = _parse_filter(tokens)
1195 current_selector.filters.append(format_filter)
1196 elif string == '(':
1197 if current_selector:
1198 raise syntax_error('Unexpected "("', start)
1199 group = _parse_format_selection(tokens, inside_group=True)
1200 current_selector = FormatSelector(GROUP, group, [])
1201 elif string == '+':
1202 video_selector = current_selector
1203 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1204 if not video_selector or not audio_selector:
1205 raise syntax_error('"+" must be between two format selectors', start)
1206 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1207 else:
1208 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1209 elif type == tokenize.ENDMARKER:
1210 break
1211 if current_selector:
1212 selectors.append(current_selector)
1213 return selectors
1214
1215 def _build_selector_function(selector):
1216 if isinstance(selector, list):
1217 fs = [_build_selector_function(s) for s in selector]
1218
1219 def selector_function(ctx):
1220 for f in fs:
1221 for format in f(ctx):
1222 yield format
1223 return selector_function
1224 elif selector.type == GROUP:
1225 selector_function = _build_selector_function(selector.selector)
1226 elif selector.type == PICKFIRST:
1227 fs = [_build_selector_function(s) for s in selector.selector]
1228
1229 def selector_function(ctx):
1230 for f in fs:
1231 picked_formats = list(f(ctx))
1232 if picked_formats:
1233 return picked_formats
1234 return []
1235 elif selector.type == SINGLE:
1236 format_spec = selector.selector
1237
1238 def selector_function(ctx):
1239 formats = list(ctx['formats'])
1240 if not formats:
1241 return
1242 if format_spec == 'all':
1243 for f in formats:
1244 yield f
1245 elif format_spec in ['best', 'worst', None]:
1246 format_idx = 0 if format_spec == 'worst' else -1
1247 audiovideo_formats = [
1248 f for f in formats
1249 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1250 if audiovideo_formats:
1251 yield audiovideo_formats[format_idx]
1252 # for extractors with incomplete formats (audio only (soundcloud)
1253 # or video only (imgur)) we will fallback to best/worst
1254 # {video,audio}-only format
1255 elif ctx['incomplete_formats']:
1256 yield formats[format_idx]
1257 elif format_spec == 'bestaudio':
1258 audio_formats = [
1259 f for f in formats
1260 if f.get('vcodec') == 'none']
1261 if audio_formats:
1262 yield audio_formats[-1]
1263 elif format_spec == 'worstaudio':
1264 audio_formats = [
1265 f for f in formats
1266 if f.get('vcodec') == 'none']
1267 if audio_formats:
1268 yield audio_formats[0]
1269 elif format_spec == 'bestvideo':
1270 video_formats = [
1271 f for f in formats
1272 if f.get('acodec') == 'none']
1273 if video_formats:
1274 yield video_formats[-1]
1275 elif format_spec == 'worstvideo':
1276 video_formats = [
1277 f for f in formats
1278 if f.get('acodec') == 'none']
1279 if video_formats:
1280 yield video_formats[0]
1281 else:
1282 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1283 if format_spec in extensions:
1284 filter_f = lambda f: f['ext'] == format_spec
1285 else:
1286 filter_f = lambda f: f['format_id'] == format_spec
1287 matches = list(filter(filter_f, formats))
1288 if matches:
1289 yield matches[-1]
1290 elif selector.type == MERGE:
1291 def _merge(formats_info):
1292 format_1, format_2 = [f['format_id'] for f in formats_info]
1293 # The first format must contain the video and the
1294 # second the audio
1295 if formats_info[0].get('vcodec') == 'none':
1296 self.report_error('The first format must '
1297 'contain the video, try using '
1298 '"-f %s+%s"' % (format_2, format_1))
1299 return
1300 # Formats must be opposite (video+audio)
1301 if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1302 self.report_error(
1303 'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1304 % (format_1, format_2))
1305 return
1306 output_ext = (
1307 formats_info[0]['ext']
1308 if self.params.get('merge_output_format') is None
1309 else self.params['merge_output_format'])
1310 return {
1311 'requested_formats': formats_info,
1312 'format': '%s+%s' % (formats_info[0].get('format'),
1313 formats_info[1].get('format')),
1314 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1315 formats_info[1].get('format_id')),
1316 'width': formats_info[0].get('width'),
1317 'height': formats_info[0].get('height'),
1318 'resolution': formats_info[0].get('resolution'),
1319 'fps': formats_info[0].get('fps'),
1320 'vcodec': formats_info[0].get('vcodec'),
1321 'vbr': formats_info[0].get('vbr'),
1322 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1323 'acodec': formats_info[1].get('acodec'),
1324 'abr': formats_info[1].get('abr'),
1325 'ext': output_ext,
1326 }
1327 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1328
1329 def selector_function(ctx):
1330 for pair in itertools.product(
1331 video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1332 yield _merge(pair)
1333
1334 filters = [self._build_format_filter(f) for f in selector.filters]
1335
1336 def final_selector(ctx):
1337 ctx_copy = copy.deepcopy(ctx)
1338 for _filter in filters:
1339 ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1340 return selector_function(ctx_copy)
1341 return final_selector
1342
1343 stream = io.BytesIO(format_spec.encode('utf-8'))
1344 try:
1345 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1346 except tokenize.TokenError:
1347 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1348
1349 class TokenIterator(object):
1350 def __init__(self, tokens):
1351 self.tokens = tokens
1352 self.counter = 0
1353
1354 def __iter__(self):
1355 return self
1356
1357 def __next__(self):
1358 if self.counter >= len(self.tokens):
1359 raise StopIteration()
1360 value = self.tokens[self.counter]
1361 self.counter += 1
1362 return value
1363
1364 next = __next__
1365
1366 def restore_last_token(self):
1367 self.counter -= 1
1368
1369 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1370 return _build_selector_function(parsed_selector)
1371
1372 def _calc_headers(self, info_dict):
1373 res = std_headers.copy()
1374
1375 add_headers = info_dict.get('http_headers')
1376 if add_headers:
1377 res.update(add_headers)
1378
1379 cookies = self._calc_cookies(info_dict)
1380 if cookies:
1381 res['Cookie'] = cookies
1382
1383 if 'X-Forwarded-For' not in res:
1384 x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
1385 if x_forwarded_for_ip:
1386 res['X-Forwarded-For'] = x_forwarded_for_ip
1387
1388 return res
1389
1390 def _calc_cookies(self, info_dict):
1391 pr = sanitized_Request(info_dict['url'])
1392 self.cookiejar.add_cookie_header(pr)
1393 return pr.get_header('Cookie')
1394
1395 def process_video_result(self, info_dict, download=True):
1396 assert info_dict.get('_type', 'video') == 'video'
1397
1398 if 'id' not in info_dict:
1399 raise ExtractorError('Missing "id" field in extractor result')
1400 if 'title' not in info_dict:
1401 raise ExtractorError('Missing "title" field in extractor result')
1402
1403 def report_force_conversion(field, field_not, conversion):
1404 self.report_warning(
1405 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1406 % (field, field_not, conversion))
1407
1408 def sanitize_string_field(info, string_field):
1409 field = info.get(string_field)
1410 if field is None or isinstance(field, compat_str):
1411 return
1412 report_force_conversion(string_field, 'a string', 'string')
1413 info[string_field] = compat_str(field)
1414
1415 def sanitize_numeric_fields(info):
1416 for numeric_field in self._NUMERIC_FIELDS:
1417 field = info.get(numeric_field)
1418 if field is None or isinstance(field, compat_numeric_types):
1419 continue
1420 report_force_conversion(numeric_field, 'numeric', 'int')
1421 info[numeric_field] = int_or_none(field)
1422
1423 sanitize_string_field(info_dict, 'id')
1424 sanitize_numeric_fields(info_dict)
1425
1426 if 'playlist' not in info_dict:
1427 # It isn't part of a playlist
1428 info_dict['playlist'] = None
1429 info_dict['playlist_index'] = None
1430
1431 thumbnails = info_dict.get('thumbnails')
1432 if thumbnails is None:
1433 thumbnail = info_dict.get('thumbnail')
1434 if thumbnail:
1435 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1436 if thumbnails:
1437 thumbnails.sort(key=lambda t: (
1438 t.get('preference') if t.get('preference') is not None else -1,
1439 t.get('width') if t.get('width') is not None else -1,
1440 t.get('height') if t.get('height') is not None else -1,
1441 t.get('id') if t.get('id') is not None else '', t.get('url')))
1442 for i, t in enumerate(thumbnails):
1443 t['url'] = sanitize_url(t['url'])
1444 if t.get('width') and t.get('height'):
1445 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1446 if t.get('id') is None:
1447 t['id'] = '%d' % i
1448
1449 if self.params.get('list_thumbnails'):
1450 self.list_thumbnails(info_dict)
1451 return
1452
1453 thumbnail = info_dict.get('thumbnail')
1454 if thumbnail:
1455 info_dict['thumbnail'] = sanitize_url(thumbnail)
1456 elif thumbnails:
1457 info_dict['thumbnail'] = thumbnails[-1]['url']
1458
1459 if 'display_id' not in info_dict and 'id' in info_dict:
1460 info_dict['display_id'] = info_dict['id']
1461
1462 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1463 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1464 # see http://bugs.python.org/issue1646728)
1465 try:
1466 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1467 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1468 except (ValueError, OverflowError, OSError):
1469 pass
1470
1471 # Auto generate title fields corresponding to the *_number fields when missing
1472 # in order to always have clean titles. This is very common for TV series.
1473 for field in ('chapter', 'season', 'episode'):
1474 if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1475 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1476
1477 subtitles = info_dict.get('subtitles')
1478 if subtitles:
1479 for _, subtitle in subtitles.items():
1480 for subtitle_format in subtitle:
1481 if subtitle_format.get('url'):
1482 subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1483 if subtitle_format.get('ext') is None:
1484 subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1485
1486 if self.params.get('listsubtitles', False):
1487 if 'automatic_captions' in info_dict:
1488 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1489 self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1490 return
1491 info_dict['requested_subtitles'] = self.process_subtitles(
1492 info_dict['id'], subtitles,
1493 info_dict.get('automatic_captions'))
1494
1495 # We now pick which formats have to be downloaded
1496 if info_dict.get('formats') is None:
1497 # There's only one format available
1498 formats = [info_dict]
1499 else:
1500 formats = info_dict['formats']
1501
1502 if not formats:
1503 raise ExtractorError('No video formats found!')
1504
1505 def is_wellformed(f):
1506 url = f.get('url')
1507 if not url:
1508 self.report_warning(
1509 '"url" field is missing or empty - skipping format, '
1510 'there is an error in extractor')
1511 return False
1512 if isinstance(url, bytes):
1513 sanitize_string_field(f, 'url')
1514 return True
1515
1516 # Filter out malformed formats for better extraction robustness
1517 formats = list(filter(is_wellformed, formats))
1518
1519 formats_dict = {}
1520
1521 # We check that all the formats have the format and format_id fields
1522 for i, format in enumerate(formats):
1523 sanitize_string_field(format, 'format_id')
1524 sanitize_numeric_fields(format)
1525 format['url'] = sanitize_url(format['url'])
1526 if not format.get('format_id'):
1527 format['format_id'] = compat_str(i)
1528 else:
1529 # Sanitize format_id from characters used in format selector expression
1530 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1531 format_id = format['format_id']
1532 if format_id not in formats_dict:
1533 formats_dict[format_id] = []
1534 formats_dict[format_id].append(format)
1535
1536 # Make sure all formats have unique format_id
1537 for format_id, ambiguous_formats in formats_dict.items():
1538 if len(ambiguous_formats) > 1:
1539 for i, format in enumerate(ambiguous_formats):
1540 format['format_id'] = '%s-%d' % (format_id, i)
1541
1542 for i, format in enumerate(formats):
1543 if format.get('format') is None:
1544 format['format'] = '{id} - {res}{note}'.format(
1545 id=format['format_id'],
1546 res=self.format_resolution(format),
1547 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1548 )
1549 # Automatically determine file extension if missing
1550 if format.get('ext') is None:
1551 format['ext'] = determine_ext(format['url']).lower()
1552 # Automatically determine protocol if missing (useful for format
1553 # selection purposes)
1554 if format.get('protocol') is None:
1555 format['protocol'] = determine_protocol(format)
1556 # Add HTTP headers, so that external programs can use them from the
1557 # json output
1558 full_format_info = info_dict.copy()
1559 full_format_info.update(format)
1560 format['http_headers'] = self._calc_headers(full_format_info)
1561 # Remove private housekeeping stuff
1562 if '__x_forwarded_for_ip' in info_dict:
1563 del info_dict['__x_forwarded_for_ip']
1564
1565 # TODO Central sorting goes here
1566
1567 if formats[0] is not info_dict:
1568 # only set the 'formats' fields if the original info_dict list them
1569 # otherwise we end up with a circular reference, the first (and unique)
1570 # element in the 'formats' field in info_dict is info_dict itself,
1571 # which can't be exported to json
1572 info_dict['formats'] = formats
1573 if self.params.get('listformats'):
1574 self.list_formats(info_dict)
1575 return
1576
1577 req_format = self.params.get('format')
1578 if req_format is None:
1579 req_format = self._default_format_spec(info_dict, download=download)
1580 if self.params.get('verbose'):
1581 self.to_stdout('[debug] Default format spec: %s' % req_format)
1582
1583 format_selector = self.build_format_selector(req_format)
1584
1585 # While in format selection we may need to have an access to the original
1586 # format set in order to calculate some metrics or do some processing.
1587 # For now we need to be able to guess whether original formats provided
1588 # by extractor are incomplete or not (i.e. whether extractor provides only
1589 # video-only or audio-only formats) for proper formats selection for
1590 # extractors with such incomplete formats (see
1591 # https://github.com/rg3/youtube-dl/pull/5556).
1592 # Since formats may be filtered during format selection and may not match
1593 # the original formats the results may be incorrect. Thus original formats
1594 # or pre-calculated metrics should be passed to format selection routines
1595 # as well.
1596 # We will pass a context object containing all necessary additional data
1597 # instead of just formats.
1598 # This fixes incorrect format selection issue (see
1599 # https://github.com/rg3/youtube-dl/issues/10083).
1600 incomplete_formats = (
1601 # All formats are video-only or
1602 all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1603 # all formats are audio-only
1604 all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1605
1606 ctx = {
1607 'formats': formats,
1608 'incomplete_formats': incomplete_formats,
1609 }
1610
1611 formats_to_download = list(format_selector(ctx))
1612 if not formats_to_download:
1613 raise ExtractorError('requested format not available',
1614 expected=True)
1615
1616 if download:
1617 if len(formats_to_download) > 1:
1618 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1619 for format in formats_to_download:
1620 new_info = dict(info_dict)
1621 new_info.update(format)
1622 self.process_info(new_info)
1623 # We update the info dict with the best quality format (backwards compatibility)
1624 info_dict.update(formats_to_download[-1])
1625 return info_dict
1626
1627 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1628 """Select the requested subtitles and their format"""
1629 available_subs = {}
1630 if normal_subtitles and self.params.get('writesubtitles'):
1631 available_subs.update(normal_subtitles)
1632 if automatic_captions and self.params.get('writeautomaticsub'):
1633 for lang, cap_info in automatic_captions.items():
1634 if lang not in available_subs:
1635 available_subs[lang] = cap_info
1636
1637 if (not self.params.get('writesubtitles') and not
1638 self.params.get('writeautomaticsub') or not
1639 available_subs):
1640 return None
1641
1642 if self.params.get('allsubtitles', False):
1643 requested_langs = available_subs.keys()
1644 else:
1645 if self.params.get('subtitleslangs', False):
1646 requested_langs = self.params.get('subtitleslangs')
1647 elif 'en' in available_subs:
1648 requested_langs = ['en']
1649 else:
1650 requested_langs = [list(available_subs.keys())[0]]
1651
1652 formats_query = self.params.get('subtitlesformat', 'best')
1653 formats_preference = formats_query.split('/') if formats_query else []
1654 subs = {}
1655 for lang in requested_langs:
1656 formats = available_subs.get(lang)
1657 if formats is None:
1658 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1659 continue
1660 for ext in formats_preference:
1661 if ext == 'best':
1662 f = formats[-1]
1663 break
1664 matches = list(filter(lambda f: f['ext'] == ext, formats))
1665 if matches:
1666 f = matches[-1]
1667 break
1668 else:
1669 f = formats[-1]
1670 self.report_warning(
1671 'No subtitle format found matching "%s" for language %s, '
1672 'using %s' % (formats_query, lang, f['ext']))
1673 subs[lang] = f
1674 return subs
1675
1676 def process_info(self, info_dict):
1677 """Process a single resolved IE result."""
1678
1679 assert info_dict.get('_type', 'video') == 'video'
1680
1681 max_downloads = self.params.get('max_downloads')
1682 if max_downloads is not None:
1683 if self._num_downloads >= int(max_downloads):
1684 raise MaxDownloadsReached()
1685
1686 info_dict['fulltitle'] = info_dict['title']
1687 if len(info_dict['title']) > 200:
1688 info_dict['title'] = info_dict['title'][:197] + '...'
1689
1690 if 'format' not in info_dict:
1691 info_dict['format'] = info_dict['ext']
1692
1693 reason = self._match_entry(info_dict, incomplete=False)
1694 if reason is not None:
1695 self.to_screen('[download] ' + reason)
1696 return
1697
1698 self._num_downloads += 1
1699
1700 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1701
1702 # Forced printings
1703 if self.params.get('forcetitle', False):
1704 self.to_stdout(info_dict['fulltitle'])
1705 if self.params.get('forceid', False):
1706 self.to_stdout(info_dict['id'])
1707 if self.params.get('forceurl', False):
1708 if info_dict.get('requested_formats') is not None:
1709 for f in info_dict['requested_formats']:
1710 self.to_stdout(f['url'] + f.get('play_path', ''))
1711 else:
1712 # For RTMP URLs, also include the playpath
1713 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1714 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1715 self.to_stdout(info_dict['thumbnail'])
1716 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1717 self.to_stdout(info_dict['description'])
1718 if self.params.get('forcefilename', False) and filename is not None:
1719 self.to_stdout(filename)
1720 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1721 self.to_stdout(formatSeconds(info_dict['duration']))
1722 if self.params.get('forceformat', False):
1723 self.to_stdout(info_dict['format'])
1724 if self.params.get('forcejson', False):
1725 self.to_stdout(json.dumps(info_dict))
1726
1727 # Do nothing else if in simulate mode
1728 if self.params.get('simulate', False):
1729 return
1730
1731 if filename is None:
1732 return
1733
1734 def ensure_dir_exists(path):
1735 try:
1736 dn = os.path.dirname(path)
1737 if dn and not os.path.exists(dn):
1738 os.makedirs(dn)
1739 return True
1740 except (OSError, IOError) as err:
1741 self.report_error('unable to create directory ' + error_to_compat_str(err))
1742 return False
1743
1744 if not ensure_dir_exists(sanitize_path(encodeFilename(filename))):
1745 return
1746
1747 if self.params.get('writedescription', False):
1748 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1749 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1750 self.to_screen('[info] Video description is already present')
1751 elif info_dict.get('description') is None:
1752 self.report_warning('There\'s no description to write.')
1753 else:
1754 try:
1755 self.to_screen('[info] Writing video description to: ' + descfn)
1756 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1757 descfile.write(info_dict['description'])
1758 except (OSError, IOError):
1759 self.report_error('Cannot write description file ' + descfn)
1760 return
1761
1762 if self.params.get('writeannotations', False):
1763 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1764 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1765 self.to_screen('[info] Video annotations are already present')
1766 else:
1767 try:
1768 self.to_screen('[info] Writing video annotations to: ' + annofn)
1769 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1770 annofile.write(info_dict['annotations'])
1771 except (KeyError, TypeError):
1772 self.report_warning('There are no annotations to write.')
1773 except (OSError, IOError):
1774 self.report_error('Cannot write annotations file: ' + annofn)
1775 return
1776
1777 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1778 self.params.get('writeautomaticsub')])
1779
1780 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1781 # subtitles download errors are already managed as troubles in relevant IE
1782 # that way it will silently go on when used with unsupporting IE
1783 subtitles = info_dict['requested_subtitles']
1784 ie = self.get_info_extractor(info_dict['extractor_key'])
1785 for sub_lang, sub_info in subtitles.items():
1786 sub_format = sub_info['ext']
1787 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1788 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1789 self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
1790 else:
1791 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1792 if sub_info.get('data') is not None:
1793 try:
1794 # Use newline='' to prevent conversion of newline characters
1795 # See https://github.com/rg3/youtube-dl/issues/10268
1796 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1797 subfile.write(sub_info['data'])
1798 except (OSError, IOError):
1799 self.report_error('Cannot write subtitles file ' + sub_filename)
1800 return
1801 else:
1802 try:
1803 sub_data = ie._request_webpage(
1804 sub_info['url'], info_dict['id'], note=False).read()
1805 with io.open(encodeFilename(sub_filename), 'wb') as subfile:
1806 subfile.write(sub_data)
1807 except (ExtractorError, IOError, OSError, ValueError) as err:
1808 self.report_warning('Unable to download subtitle for "%s": %s' %
1809 (sub_lang, error_to_compat_str(err)))
1810 continue
1811
1812 if self.params.get('writeinfojson', False):
1813 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1814 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1815 self.to_screen('[info] Video description metadata is already present')
1816 else:
1817 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1818 try:
1819 write_json_file(self.filter_requested_info(info_dict), infofn)
1820 except (OSError, IOError):
1821 self.report_error('Cannot write metadata to JSON file ' + infofn)
1822 return
1823
1824 self._write_thumbnails(info_dict, filename)
1825
1826 if not self.params.get('skip_download', False):
1827 try:
1828 def dl(name, info):
1829 fd = get_suitable_downloader(info, self.params)(self, self.params)
1830 for ph in self._progress_hooks:
1831 fd.add_progress_hook(ph)
1832 if self.params.get('verbose'):
1833 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1834 return fd.download(name, info)
1835
1836 if info_dict.get('requested_formats') is not None:
1837 downloaded = []
1838 success = True
1839 merger = FFmpegMergerPP(self)
1840 if not merger.available:
1841 postprocessors = []
1842 self.report_warning('You have requested multiple '
1843 'formats but ffmpeg or avconv are not installed.'
1844 ' The formats won\'t be merged.')
1845 else:
1846 postprocessors = [merger]
1847
1848 def compatible_formats(formats):
1849 video, audio = formats
1850 # Check extension
1851 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1852 if video_ext and audio_ext:
1853 COMPATIBLE_EXTS = (
1854 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1855 ('webm')
1856 )
1857 for exts in COMPATIBLE_EXTS:
1858 if video_ext in exts and audio_ext in exts:
1859 return True
1860 # TODO: Check acodec/vcodec
1861 return False
1862
1863 filename_real_ext = os.path.splitext(filename)[1][1:]
1864 filename_wo_ext = (
1865 os.path.splitext(filename)[0]
1866 if filename_real_ext == info_dict['ext']
1867 else filename)
1868 requested_formats = info_dict['requested_formats']
1869 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1870 info_dict['ext'] = 'mkv'
1871 self.report_warning(
1872 'Requested formats are incompatible for merge and will be merged into mkv.')
1873 # Ensure filename always has a correct extension for successful merge
1874 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1875 if os.path.exists(encodeFilename(filename)):
1876 self.to_screen(
1877 '[download] %s has already been downloaded and '
1878 'merged' % filename)
1879 else:
1880 for f in requested_formats:
1881 new_info = dict(info_dict)
1882 new_info.update(f)
1883 fname = prepend_extension(
1884 self.prepare_filename(new_info),
1885 'f%s' % f['format_id'], new_info['ext'])
1886 if not ensure_dir_exists(fname):
1887 return
1888 downloaded.append(fname)
1889 partial_success = dl(fname, new_info)
1890 success = success and partial_success
1891 info_dict['__postprocessors'] = postprocessors
1892 info_dict['__files_to_merge'] = downloaded
1893 else:
1894 # Just a single file
1895 success = dl(filename, info_dict)
1896 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1897 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1898 return
1899 except (OSError, IOError) as err:
1900 raise UnavailableVideoError(err)
1901 except (ContentTooShortError, ) as err:
1902 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1903 return
1904
1905 if success and filename != '-':
1906 # Fixup content
1907 fixup_policy = self.params.get('fixup')
1908 if fixup_policy is None:
1909 fixup_policy = 'detect_or_warn'
1910
1911 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1912
1913 stretched_ratio = info_dict.get('stretched_ratio')
1914 if stretched_ratio is not None and stretched_ratio != 1:
1915 if fixup_policy == 'warn':
1916 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1917 info_dict['id'], stretched_ratio))
1918 elif fixup_policy == 'detect_or_warn':
1919 stretched_pp = FFmpegFixupStretchedPP(self)
1920 if stretched_pp.available:
1921 info_dict.setdefault('__postprocessors', [])
1922 info_dict['__postprocessors'].append(stretched_pp)
1923 else:
1924 self.report_warning(
1925 '%s: Non-uniform pixel ratio (%s). %s'
1926 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1927 else:
1928 assert fixup_policy in ('ignore', 'never')
1929
1930 if (info_dict.get('requested_formats') is None and
1931 info_dict.get('container') == 'm4a_dash'):
1932 if fixup_policy == 'warn':
1933 self.report_warning(
1934 '%s: writing DASH m4a. '
1935 'Only some players support this container.'
1936 % info_dict['id'])
1937 elif fixup_policy == 'detect_or_warn':
1938 fixup_pp = FFmpegFixupM4aPP(self)
1939 if fixup_pp.available:
1940 info_dict.setdefault('__postprocessors', [])
1941 info_dict['__postprocessors'].append(fixup_pp)
1942 else:
1943 self.report_warning(
1944 '%s: writing DASH m4a. '
1945 'Only some players support this container. %s'
1946 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1947 else:
1948 assert fixup_policy in ('ignore', 'never')
1949
1950 if (info_dict.get('protocol') == 'm3u8_native' or
1951 info_dict.get('protocol') == 'm3u8' and
1952 self.params.get('hls_prefer_native')):
1953 if fixup_policy == 'warn':
1954 self.report_warning('%s: malformed AAC bitstream detected.' % (
1955 info_dict['id']))
1956 elif fixup_policy == 'detect_or_warn':
1957 fixup_pp = FFmpegFixupM3u8PP(self)
1958 if fixup_pp.available:
1959 info_dict.setdefault('__postprocessors', [])
1960 info_dict['__postprocessors'].append(fixup_pp)
1961 else:
1962 self.report_warning(
1963 '%s: malformed AAC bitstream detected. %s'
1964 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1965 else:
1966 assert fixup_policy in ('ignore', 'never')
1967
1968 try:
1969 self.post_process(filename, info_dict)
1970 except (PostProcessingError) as err:
1971 self.report_error('postprocessing: %s' % str(err))
1972 return
1973 self.record_download_archive(info_dict)
1974
1975 def download(self, url_list):
1976 """Download a given list of URLs."""
1977 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1978 if (len(url_list) > 1 and
1979 outtmpl != '-' and
1980 '%' not in outtmpl and
1981 self.params.get('max_downloads') != 1):
1982 raise SameFileError(outtmpl)
1983
1984 for url in url_list:
1985 try:
1986 # It also downloads the videos
1987 res = self.extract_info(
1988 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1989 except UnavailableVideoError:
1990 self.report_error('unable to download video')
1991 except MaxDownloadsReached:
1992 self.to_screen('[info] Maximum number of downloaded files reached.')
1993 raise
1994 else:
1995 if self.params.get('dump_single_json', False):
1996 self.to_stdout(json.dumps(res))
1997
1998 return self._download_retcode
1999
2000 def download_with_info_file(self, info_filename):
2001 with contextlib.closing(fileinput.FileInput(
2002 [info_filename], mode='r',
2003 openhook=fileinput.hook_encoded('utf-8'))) as f:
2004 # FileInput doesn't have a read method, we can't call json.load
2005 info = self.filter_requested_info(json.loads('\n'.join(f)))
2006 try:
2007 self.process_ie_result(info, download=True)
2008 except DownloadError:
2009 webpage_url = info.get('webpage_url')
2010 if webpage_url is not None:
2011 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
2012 return self.download([webpage_url])
2013 else:
2014 raise
2015 return self._download_retcode
2016
2017 @staticmethod
2018 def filter_requested_info(info_dict):
2019 return dict(
2020 (k, v) for k, v in info_dict.items()
2021 if k not in ['requested_formats', 'requested_subtitles'])
2022
2023 def post_process(self, filename, ie_info):
2024 """Run all the postprocessors on the given file."""
2025 info = dict(ie_info)
2026 info['filepath'] = filename
2027 pps_chain = []
2028 if ie_info.get('__postprocessors') is not None:
2029 pps_chain.extend(ie_info['__postprocessors'])
2030 pps_chain.extend(self._pps)
2031 for pp in pps_chain:
2032 files_to_delete = []
2033 try:
2034 files_to_delete, info = pp.run(info)
2035 except PostProcessingError as e:
2036 self.report_error(e.msg)
2037 if files_to_delete and not self.params.get('keepvideo', False):
2038 for old_filename in files_to_delete:
2039 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
2040 try:
2041 os.remove(encodeFilename(old_filename))
2042 except (IOError, OSError):
2043 self.report_warning('Unable to remove downloaded original file')
2044
2045 def _make_archive_id(self, info_dict):
2046 # Future-proof against any change in case
2047 # and backwards compatibility with prior versions
2048 extractor = info_dict.get('extractor_key')
2049 if extractor is None:
2050 if 'id' in info_dict:
2051 extractor = info_dict.get('ie_key') # key in a playlist
2052 if extractor is None:
2053 return None # Incomplete video information
2054 return extractor.lower() + ' ' + info_dict['id']
2055
2056 def in_download_archive(self, info_dict):
2057 fn = self.params.get('download_archive')
2058 if fn is None:
2059 return False
2060
2061 vid_id = self._make_archive_id(info_dict)
2062 if vid_id is None:
2063 return False # Incomplete video information
2064
2065 try:
2066 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
2067 for line in archive_file:
2068 if line.strip() == vid_id:
2069 return True
2070 except IOError as ioe:
2071 if ioe.errno != errno.ENOENT:
2072 raise
2073 return False
2074
2075 def record_download_archive(self, info_dict):
2076 fn = self.params.get('download_archive')
2077 if fn is None:
2078 return
2079 vid_id = self._make_archive_id(info_dict)
2080 assert vid_id
2081 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
2082 archive_file.write(vid_id + '\n')
2083
2084 @staticmethod
2085 def format_resolution(format, default='unknown'):
2086 if format.get('vcodec') == 'none':
2087 return 'audio only'
2088 if format.get('resolution') is not None:
2089 return format['resolution']
2090 if format.get('height') is not None:
2091 if format.get('width') is not None:
2092 res = '%sx%s' % (format['width'], format['height'])
2093 else:
2094 res = '%sp' % format['height']
2095 elif format.get('width') is not None:
2096 res = '%dx?' % format['width']
2097 else:
2098 res = default
2099 return res
2100
2101 def _format_note(self, fdict):
2102 res = ''
2103 if fdict.get('ext') in ['f4f', 'f4m']:
2104 res += '(unsupported) '
2105 if fdict.get('language'):
2106 if res:
2107 res += ' '
2108 res += '[%s] ' % fdict['language']
2109 if fdict.get('format_note') is not None:
2110 res += fdict['format_note'] + ' '
2111 if fdict.get('tbr') is not None:
2112 res += '%4dk ' % fdict['tbr']
2113 if fdict.get('container') is not None:
2114 if res:
2115 res += ', '
2116 res += '%s container' % fdict['container']
2117 if (fdict.get('vcodec') is not None and
2118 fdict.get('vcodec') != 'none'):
2119 if res:
2120 res += ', '
2121 res += fdict['vcodec']
2122 if fdict.get('vbr') is not None:
2123 res += '@'
2124 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
2125 res += 'video@'
2126 if fdict.get('vbr') is not None:
2127 res += '%4dk' % fdict['vbr']
2128 if fdict.get('fps') is not None:
2129 if res:
2130 res += ', '
2131 res += '%sfps' % fdict['fps']
2132 if fdict.get('acodec') is not None:
2133 if res:
2134 res += ', '
2135 if fdict['acodec'] == 'none':
2136 res += 'video only'
2137 else:
2138 res += '%-5s' % fdict['acodec']
2139 elif fdict.get('abr') is not None:
2140 if res:
2141 res += ', '
2142 res += 'audio'
2143 if fdict.get('abr') is not None:
2144 res += '@%3dk' % fdict['abr']
2145 if fdict.get('asr') is not None:
2146 res += ' (%5dHz)' % fdict['asr']
2147 if fdict.get('filesize') is not None:
2148 if res:
2149 res += ', '
2150 res += format_bytes(fdict['filesize'])
2151 elif fdict.get('filesize_approx') is not None:
2152 if res:
2153 res += ', '
2154 res += '~' + format_bytes(fdict['filesize_approx'])
2155 return res
2156
2157 def list_formats(self, info_dict):
2158 formats = info_dict.get('formats', [info_dict])
2159 table = [
2160 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
2161 for f in formats
2162 if f.get('preference') is None or f['preference'] >= -1000]
2163 if len(formats) > 1:
2164 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2165
2166 header_line = ['format code', 'extension', 'resolution', 'note']
2167 self.to_screen(
2168 '[info] Available formats for %s:\n%s' %
2169 (info_dict['id'], render_table(header_line, table)))
2170
2171 def list_thumbnails(self, info_dict):
2172 thumbnails = info_dict.get('thumbnails')
2173 if not thumbnails:
2174 self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
2175 return
2176
2177 self.to_screen(
2178 '[info] Thumbnails for %s:' % info_dict['id'])
2179 self.to_screen(render_table(
2180 ['ID', 'width', 'height', 'URL'],
2181 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
2182
2183 def list_subtitles(self, video_id, subtitles, name='subtitles'):
2184 if not subtitles:
2185 self.to_screen('%s has no %s' % (video_id, name))
2186 return
2187 self.to_screen(
2188 'Available %s for %s:' % (name, video_id))
2189 self.to_screen(render_table(
2190 ['Language', 'formats'],
2191 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2192 for lang, formats in subtitles.items()]))
2193
2194 def urlopen(self, req):
2195 """ Start an HTTP download """
2196 if isinstance(req, compat_basestring):
2197 req = sanitized_Request(req)
2198 return self._opener.open(req, timeout=self._socket_timeout)
2199
2200 def print_debug_header(self):
2201 if not self.params.get('verbose'):
2202 return
2203
2204 if type('') is not compat_str:
2205 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2206 self.report_warning(
2207 'Your Python is broken! Update to a newer and supported version')
2208
2209 stdout_encoding = getattr(
2210 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2211 encoding_str = (
2212 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2213 locale.getpreferredencoding(),
2214 sys.getfilesystemencoding(),
2215 stdout_encoding,
2216 self.get_encoding()))
2217 write_string(encoding_str, encoding=None)
2218
2219 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2220 if _LAZY_LOADER:
2221 self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2222 try:
2223 sp = subprocess.Popen(
2224 ['git', 'rev-parse', '--short', 'HEAD'],
2225 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2226 cwd=os.path.dirname(os.path.abspath(__file__)))
2227 out, err = sp.communicate()
2228 out = out.decode().strip()
2229 if re.match('[0-9a-f]+', out):
2230 self._write_string('[debug] Git HEAD: ' + out + '\n')
2231 except Exception:
2232 try:
2233 sys.exc_clear()
2234 except Exception:
2235 pass
2236 self._write_string('[debug] Python version %s - %s\n' % (
2237 platform.python_version(), platform_name()))
2238
2239 exe_versions = FFmpegPostProcessor.get_versions(self)
2240 exe_versions['rtmpdump'] = rtmpdump_version()
2241 exe_versions['phantomjs'] = PhantomJSwrapper._version()
2242 exe_str = ', '.join(
2243 '%s %s' % (exe, v)
2244 for exe, v in sorted(exe_versions.items())
2245 if v
2246 )
2247 if not exe_str:
2248 exe_str = 'none'
2249 self._write_string('[debug] exe versions: %s\n' % exe_str)
2250
2251 proxy_map = {}
2252 for handler in self._opener.handlers:
2253 if hasattr(handler, 'proxies'):
2254 proxy_map.update(handler.proxies)
2255 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2256
2257 if self.params.get('call_home', False):
2258 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2259 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2260 latest_version = self.urlopen(
2261 'https://yt-dl.org/latest/version').read().decode('utf-8')
2262 if version_tuple(latest_version) > version_tuple(__version__):
2263 self.report_warning(
2264 'You are using an outdated version (newest version: %s)! '
2265 'See https://yt-dl.org/update if you need help updating.' %
2266 latest_version)
2267
2268 def _setup_opener(self):
2269 timeout_val = self.params.get('socket_timeout')
2270 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2271
2272 opts_cookiefile = self.params.get('cookiefile')
2273 opts_proxy = self.params.get('proxy')
2274
2275 if opts_cookiefile is None:
2276 self.cookiejar = compat_cookiejar.CookieJar()
2277 else:
2278 opts_cookiefile = expand_path(opts_cookiefile)
2279 self.cookiejar = compat_cookiejar.MozillaCookieJar(
2280 opts_cookiefile)
2281 if os.access(opts_cookiefile, os.R_OK):
2282 self.cookiejar.load()
2283
2284 cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2285 if opts_proxy is not None:
2286 if opts_proxy == '':
2287 proxies = {}
2288 else:
2289 proxies = {'http': opts_proxy, 'https': opts_proxy}
2290 else:
2291 proxies = compat_urllib_request.getproxies()
2292 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2293 if 'http' in proxies and 'https' not in proxies:
2294 proxies['https'] = proxies['http']
2295 proxy_handler = PerRequestProxyHandler(proxies)
2296
2297 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2298 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2299 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2300 data_handler = compat_urllib_request_DataHandler()
2301
2302 # When passing our own FileHandler instance, build_opener won't add the
2303 # default FileHandler and allows us to disable the file protocol, which
2304 # can be used for malicious purposes (see
2305 # https://github.com/rg3/youtube-dl/issues/8227)
2306 file_handler = compat_urllib_request.FileHandler()
2307
2308 def file_open(*args, **kwargs):
2309 raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2310 file_handler.file_open = file_open
2311
2312 opener = compat_urllib_request.build_opener(
2313 proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2314
2315 # Delete the default user-agent header, which would otherwise apply in
2316 # cases where our custom HTTP handler doesn't come into play
2317 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2318 opener.addheaders = []
2319 self._opener = opener
2320
2321 def encode(self, s):
2322 if isinstance(s, bytes):
2323 return s # Already encoded
2324
2325 try:
2326 return s.encode(self.get_encoding())
2327 except UnicodeEncodeError as err:
2328 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2329 raise
2330
2331 def get_encoding(self):
2332 encoding = self.params.get('encoding')
2333 if encoding is None:
2334 encoding = preferredencoding()
2335 return encoding
2336
2337 def _write_thumbnails(self, info_dict, filename):
2338 if self.params.get('writethumbnail', False):
2339 thumbnails = info_dict.get('thumbnails')
2340 if thumbnails:
2341 thumbnails = [thumbnails[-1]]
2342 elif self.params.get('write_all_thumbnails', False):
2343 thumbnails = info_dict.get('thumbnails')
2344 else:
2345 return
2346
2347 if not thumbnails:
2348 # No thumbnails present, so return immediately
2349 return
2350
2351 for t in thumbnails:
2352 thumb_ext = determine_ext(t['url'], 'jpg')
2353 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2354 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2355 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2356
2357 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2358 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2359 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2360 else:
2361 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2362 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2363 try:
2364 uf = self.urlopen(t['url'])
2365 with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2366 shutil.copyfileobj(uf, thumbf)
2367 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2368 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2369 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2370 self.report_warning('Unable to download thumbnail "%s": %s' %
2371 (t['url'], error_to_compat_str(err)))