]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py
Update upstream source from tag 'upstream/2019.06.08'
[youtubedl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 from __future__ import absolute_import, unicode_literals
5
6 import collections
7 import contextlib
8 import copy
9 import datetime
10 import errno
11 import fileinput
12 import io
13 import itertools
14 import json
15 import locale
16 import operator
17 import os
18 import platform
19 import re
20 import shutil
21 import subprocess
22 import socket
23 import sys
24 import time
25 import tokenize
26 import traceback
27 import random
28
29 from string import ascii_letters
30
31 from .compat import (
32 compat_basestring,
33 compat_cookiejar,
34 compat_get_terminal_size,
35 compat_http_client,
36 compat_kwargs,
37 compat_numeric_types,
38 compat_os_name,
39 compat_str,
40 compat_tokenize_tokenize,
41 compat_urllib_error,
42 compat_urllib_request,
43 compat_urllib_request_DataHandler,
44 )
45 from .utils import (
46 age_restricted,
47 args_to_str,
48 ContentTooShortError,
49 date_from_str,
50 DateRange,
51 DEFAULT_OUTTMPL,
52 determine_ext,
53 determine_protocol,
54 DownloadError,
55 encode_compat_str,
56 encodeFilename,
57 error_to_compat_str,
58 expand_path,
59 ExtractorError,
60 format_bytes,
61 formatSeconds,
62 GeoRestrictedError,
63 int_or_none,
64 ISO3166Utils,
65 locked_file,
66 make_HTTPS_handler,
67 MaxDownloadsReached,
68 orderedSet,
69 PagedList,
70 parse_filesize,
71 PerRequestProxyHandler,
72 platform_name,
73 PostProcessingError,
74 preferredencoding,
75 prepend_extension,
76 register_socks_protocols,
77 render_table,
78 replace_extension,
79 SameFileError,
80 sanitize_filename,
81 sanitize_path,
82 sanitize_url,
83 sanitized_Request,
84 std_headers,
85 str_or_none,
86 subtitles_filename,
87 UnavailableVideoError,
88 url_basename,
89 version_tuple,
90 write_json_file,
91 write_string,
92 YoutubeDLCookieJar,
93 YoutubeDLCookieProcessor,
94 YoutubeDLHandler,
95 )
96 from .cache import Cache
97 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
98 from .extractor.openload import PhantomJSwrapper
99 from .downloader import get_suitable_downloader
100 from .downloader.rtmp import rtmpdump_version
101 from .postprocessor import (
102 FFmpegFixupM3u8PP,
103 FFmpegFixupM4aPP,
104 FFmpegFixupStretchedPP,
105 FFmpegMergerPP,
106 FFmpegPostProcessor,
107 get_postprocessor,
108 )
109 from .version import __version__
110
111 if compat_os_name == 'nt':
112 import ctypes
113
114
115 class YoutubeDL(object):
116 """YoutubeDL class.
117
118 YoutubeDL objects are the ones responsible of downloading the
119 actual video file and writing it to disk if the user has requested
120 it, among some other tasks. In most cases there should be one per
121 program. As, given a video URL, the downloader doesn't know how to
122 extract all the needed information, task that InfoExtractors do, it
123 has to pass the URL to one of them.
124
125 For this, YoutubeDL objects have a method that allows
126 InfoExtractors to be registered in a given order. When it is passed
127 a URL, the YoutubeDL object handles it to the first InfoExtractor it
128 finds that reports being able to handle it. The InfoExtractor extracts
129 all the information about the video or videos the URL refers to, and
130 YoutubeDL process the extracted information, possibly using a File
131 Downloader to download the video.
132
133 YoutubeDL objects accept a lot of parameters. In order not to saturate
134 the object constructor with arguments, it receives a dictionary of
135 options instead. These options are available through the params
136 attribute for the InfoExtractors to use. The YoutubeDL also
137 registers itself as the downloader in charge for the InfoExtractors
138 that are added to it, so this is a "mutual registration".
139
140 Available options:
141
142 username: Username for authentication purposes.
143 password: Password for authentication purposes.
144 videopassword: Password for accessing a video.
145 ap_mso: Adobe Pass multiple-system operator identifier.
146 ap_username: Multiple-system operator account username.
147 ap_password: Multiple-system operator account password.
148 usenetrc: Use netrc for authentication instead.
149 verbose: Print additional info to stdout.
150 quiet: Do not print messages to stdout.
151 no_warnings: Do not print out anything for warnings.
152 forceurl: Force printing final URL.
153 forcetitle: Force printing title.
154 forceid: Force printing ID.
155 forcethumbnail: Force printing thumbnail URL.
156 forcedescription: Force printing description.
157 forcefilename: Force printing final filename.
158 forceduration: Force printing duration.
159 forcejson: Force printing info_dict as JSON.
160 dump_single_json: Force printing the info_dict of the whole playlist
161 (or video) as a single JSON line.
162 simulate: Do not download the video files.
163 format: Video format code. See options.py for more information.
164 outtmpl: Template for output names.
165 restrictfilenames: Do not allow "&" and spaces in file names
166 ignoreerrors: Do not stop on download errors.
167 force_generic_extractor: Force downloader to use the generic extractor
168 nooverwrites: Prevent overwriting files.
169 playliststart: Playlist item to start at.
170 playlistend: Playlist item to end at.
171 playlist_items: Specific indices of playlist to download.
172 playlistreverse: Download playlist items in reverse order.
173 playlistrandom: Download playlist items in random order.
174 matchtitle: Download only matching titles.
175 rejecttitle: Reject downloads for matching titles.
176 logger: Log messages to a logging.Logger instance.
177 logtostderr: Log messages to stderr instead of stdout.
178 writedescription: Write the video description to a .description file
179 writeinfojson: Write the video description to a .info.json file
180 writeannotations: Write the video annotations to a .annotations.xml file
181 writethumbnail: Write the thumbnail image to a file
182 write_all_thumbnails: Write all thumbnail formats to files
183 writesubtitles: Write the video subtitles to a file
184 writeautomaticsub: Write the automatically generated subtitles to a file
185 allsubtitles: Downloads all the subtitles of the video
186 (requires writesubtitles or writeautomaticsub)
187 listsubtitles: Lists all available subtitles for the video
188 subtitlesformat: The format code for subtitles
189 subtitleslangs: List of languages of the subtitles to download
190 keepvideo: Keep the video file after post-processing
191 daterange: A DateRange object, download only if the upload_date is in the range.
192 skip_download: Skip the actual download of the video file
193 cachedir: Location of the cache files in the filesystem.
194 False to disable filesystem cache.
195 noplaylist: Download single video instead of a playlist if in doubt.
196 age_limit: An integer representing the user's age in years.
197 Unsuitable videos for the given age are skipped.
198 min_views: An integer representing the minimum view count the video
199 must have in order to not be skipped.
200 Videos without view count information are always
201 downloaded. None for no limit.
202 max_views: An integer representing the maximum view count.
203 Videos that are more popular than that are not
204 downloaded.
205 Videos without view count information are always
206 downloaded. None for no limit.
207 download_archive: File name of a file where all downloads are recorded.
208 Videos already present in the file are not downloaded
209 again.
210 cookiefile: File name where cookies should be read from and dumped to.
211 nocheckcertificate:Do not verify SSL certificates
212 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
213 At the moment, this is only supported by YouTube.
214 proxy: URL of the proxy server to use
215 geo_verification_proxy: URL of the proxy to use for IP address verification
216 on geo-restricted sites.
217 socket_timeout: Time to wait for unresponsive hosts, in seconds
218 bidi_workaround: Work around buggy terminals without bidirectional text
219 support, using fridibi
220 debug_printtraffic:Print out sent and received HTTP traffic
221 include_ads: Download ads as well
222 default_search: Prepend this string if an input url is not valid.
223 'auto' for elaborate guessing
224 encoding: Use this encoding instead of the system-specified.
225 extract_flat: Do not resolve URLs, return the immediate result.
226 Pass in 'in_playlist' to only show this behavior for
227 playlist items.
228 postprocessors: A list of dictionaries, each with an entry
229 * key: The name of the postprocessor. See
230 youtube_dl/postprocessor/__init__.py for a list.
231 as well as any further keyword arguments for the
232 postprocessor.
233 progress_hooks: A list of functions that get called on download
234 progress, with a dictionary with the entries
235 * status: One of "downloading", "error", or "finished".
236 Check this first and ignore unknown values.
237
238 If status is one of "downloading", or "finished", the
239 following properties may also be present:
240 * filename: The final filename (always present)
241 * tmpfilename: The filename we're currently writing to
242 * downloaded_bytes: Bytes on disk
243 * total_bytes: Size of the whole file, None if unknown
244 * total_bytes_estimate: Guess of the eventual file size,
245 None if unavailable.
246 * elapsed: The number of seconds since download started.
247 * eta: The estimated time in seconds, None if unknown
248 * speed: The download speed in bytes/second, None if
249 unknown
250 * fragment_index: The counter of the currently
251 downloaded video fragment.
252 * fragment_count: The number of fragments (= individual
253 files that will be merged)
254
255 Progress hooks are guaranteed to be called at least once
256 (with status "finished") if the download is successful.
257 merge_output_format: Extension to use when merging formats.
258 fixup: Automatically correct known faults of the file.
259 One of:
260 - "never": do nothing
261 - "warn": only emit a warning
262 - "detect_or_warn": check whether we can do anything
263 about it, warn otherwise (default)
264 source_address: Client-side IP address to bind to.
265 call_home: Boolean, true iff we are allowed to contact the
266 youtube-dl servers for debugging.
267 sleep_interval: Number of seconds to sleep before each download when
268 used alone or a lower bound of a range for randomized
269 sleep before each download (minimum possible number
270 of seconds to sleep) when used along with
271 max_sleep_interval.
272 max_sleep_interval:Upper bound of a range for randomized sleep before each
273 download (maximum possible number of seconds to sleep).
274 Must only be used along with sleep_interval.
275 Actual sleep time will be a random float from range
276 [sleep_interval; max_sleep_interval].
277 listformats: Print an overview of available video formats and exit.
278 list_thumbnails: Print a table of all thumbnails and exit.
279 match_filter: A function that gets called with the info_dict of
280 every video.
281 If it returns a message, the video is ignored.
282 If it returns None, the video is downloaded.
283 match_filter_func in utils.py is one example for this.
284 no_color: Do not emit color codes in output.
285 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
286 HTTP header
287 geo_bypass_country:
288 Two-letter ISO 3166-2 country code that will be used for
289 explicit geographic restriction bypassing via faking
290 X-Forwarded-For HTTP header
291 geo_bypass_ip_block:
292 IP range in CIDR notation that will be used similarly to
293 geo_bypass_country
294
295 The following options determine which downloader is picked:
296 external_downloader: Executable of the external downloader to call.
297 None or unset for standard (built-in) downloader.
298 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
299 if True, otherwise use ffmpeg/avconv if False, otherwise
300 use downloader suggested by extractor if None.
301
302 The following parameters are not used by YoutubeDL itself, they are used by
303 the downloader (see youtube_dl/downloader/common.py):
304 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
305 noresizebuffer, retries, continuedl, noprogress, consoletitle,
306 xattr_set_filesize, external_downloader_args, hls_use_mpegts,
307 http_chunk_size.
308
309 The following options are used by the post processors:
310 prefer_ffmpeg: If False, use avconv instead of ffmpeg if both are available,
311 otherwise prefer ffmpeg.
312 ffmpeg_location: Location of the ffmpeg/avconv binary; either the path
313 to the binary or its containing directory.
314 postprocessor_args: A list of additional command-line arguments for the
315 postprocessor.
316
317 The following options are used by the Youtube extractor:
318 youtube_include_dash_manifest: If True (default), DASH manifests and related
319 data will be downloaded and processed by extractor.
320 You can reduce network I/O by disabling it if you don't
321 care about DASH.
322 """
323
324 _NUMERIC_FIELDS = set((
325 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
326 'timestamp', 'upload_year', 'upload_month', 'upload_day',
327 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
328 'average_rating', 'comment_count', 'age_limit',
329 'start_time', 'end_time',
330 'chapter_number', 'season_number', 'episode_number',
331 'track_number', 'disc_number', 'release_year',
332 'playlist_index',
333 ))
334
335 params = None
336 _ies = []
337 _pps = []
338 _download_retcode = None
339 _num_downloads = None
340 _screen_file = None
341
342 def __init__(self, params=None, auto_init=True):
343 """Create a FileDownloader object with the given options."""
344 if params is None:
345 params = {}
346 self._ies = []
347 self._ies_instances = {}
348 self._pps = []
349 self._progress_hooks = []
350 self._download_retcode = 0
351 self._num_downloads = 0
352 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
353 self._err_file = sys.stderr
354 self.params = {
355 # Default parameters
356 'nocheckcertificate': False,
357 }
358 self.params.update(params)
359 self.cache = Cache(self)
360
361 def check_deprecated(param, option, suggestion):
362 if self.params.get(param) is not None:
363 self.report_warning(
364 '%s is deprecated. Use %s instead.' % (option, suggestion))
365 return True
366 return False
367
368 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
369 if self.params.get('geo_verification_proxy') is None:
370 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
371
372 check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
373 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
374 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
375
376 if params.get('bidi_workaround', False):
377 try:
378 import pty
379 master, slave = pty.openpty()
380 width = compat_get_terminal_size().columns
381 if width is None:
382 width_args = []
383 else:
384 width_args = ['-w', str(width)]
385 sp_kwargs = dict(
386 stdin=subprocess.PIPE,
387 stdout=slave,
388 stderr=self._err_file)
389 try:
390 self._output_process = subprocess.Popen(
391 ['bidiv'] + width_args, **sp_kwargs
392 )
393 except OSError:
394 self._output_process = subprocess.Popen(
395 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
396 self._output_channel = os.fdopen(master, 'rb')
397 except OSError as ose:
398 if ose.errno == errno.ENOENT:
399 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
400 else:
401 raise
402
403 if (sys.platform != 'win32'
404 and sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
405 and not params.get('restrictfilenames', False)):
406 # Unicode filesystem API will throw errors (#1474, #13027)
407 self.report_warning(
408 'Assuming --restrict-filenames since file system encoding '
409 'cannot encode all characters. '
410 'Set the LC_ALL environment variable to fix this.')
411 self.params['restrictfilenames'] = True
412
413 if isinstance(params.get('outtmpl'), bytes):
414 self.report_warning(
415 'Parameter outtmpl is bytes, but should be a unicode string. '
416 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
417
418 self._setup_opener()
419
420 if auto_init:
421 self.print_debug_header()
422 self.add_default_info_extractors()
423
424 for pp_def_raw in self.params.get('postprocessors', []):
425 pp_class = get_postprocessor(pp_def_raw['key'])
426 pp_def = dict(pp_def_raw)
427 del pp_def['key']
428 pp = pp_class(self, **compat_kwargs(pp_def))
429 self.add_post_processor(pp)
430
431 for ph in self.params.get('progress_hooks', []):
432 self.add_progress_hook(ph)
433
434 register_socks_protocols()
435
436 def warn_if_short_id(self, argv):
437 # short YouTube ID starting with dash?
438 idxs = [
439 i for i, a in enumerate(argv)
440 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
441 if idxs:
442 correct_argv = (
443 ['youtube-dl']
444 + [a for i, a in enumerate(argv) if i not in idxs]
445 + ['--'] + [argv[i] for i in idxs]
446 )
447 self.report_warning(
448 'Long argument string detected. '
449 'Use -- to separate parameters and URLs, like this:\n%s\n' %
450 args_to_str(correct_argv))
451
452 def add_info_extractor(self, ie):
453 """Add an InfoExtractor object to the end of the list."""
454 self._ies.append(ie)
455 if not isinstance(ie, type):
456 self._ies_instances[ie.ie_key()] = ie
457 ie.set_downloader(self)
458
459 def get_info_extractor(self, ie_key):
460 """
461 Get an instance of an IE with name ie_key, it will try to get one from
462 the _ies list, if there's no instance it will create a new one and add
463 it to the extractor list.
464 """
465 ie = self._ies_instances.get(ie_key)
466 if ie is None:
467 ie = get_info_extractor(ie_key)()
468 self.add_info_extractor(ie)
469 return ie
470
471 def add_default_info_extractors(self):
472 """
473 Add the InfoExtractors returned by gen_extractors to the end of the list
474 """
475 for ie in gen_extractor_classes():
476 self.add_info_extractor(ie)
477
478 def add_post_processor(self, pp):
479 """Add a PostProcessor object to the end of the chain."""
480 self._pps.append(pp)
481 pp.set_downloader(self)
482
483 def add_progress_hook(self, ph):
484 """Add the progress hook (currently only for the file downloader)"""
485 self._progress_hooks.append(ph)
486
487 def _bidi_workaround(self, message):
488 if not hasattr(self, '_output_channel'):
489 return message
490
491 assert hasattr(self, '_output_process')
492 assert isinstance(message, compat_str)
493 line_count = message.count('\n') + 1
494 self._output_process.stdin.write((message + '\n').encode('utf-8'))
495 self._output_process.stdin.flush()
496 res = ''.join(self._output_channel.readline().decode('utf-8')
497 for _ in range(line_count))
498 return res[:-len('\n')]
499
500 def to_screen(self, message, skip_eol=False):
501 """Print message to stdout if not in quiet mode."""
502 return self.to_stdout(message, skip_eol, check_quiet=True)
503
504 def _write_string(self, s, out=None):
505 write_string(s, out=out, encoding=self.params.get('encoding'))
506
507 def to_stdout(self, message, skip_eol=False, check_quiet=False):
508 """Print message to stdout if not in quiet mode."""
509 if self.params.get('logger'):
510 self.params['logger'].debug(message)
511 elif not check_quiet or not self.params.get('quiet', False):
512 message = self._bidi_workaround(message)
513 terminator = ['\n', ''][skip_eol]
514 output = message + terminator
515
516 self._write_string(output, self._screen_file)
517
518 def to_stderr(self, message):
519 """Print message to stderr."""
520 assert isinstance(message, compat_str)
521 if self.params.get('logger'):
522 self.params['logger'].error(message)
523 else:
524 message = self._bidi_workaround(message)
525 output = message + '\n'
526 self._write_string(output, self._err_file)
527
528 def to_console_title(self, message):
529 if not self.params.get('consoletitle', False):
530 return
531 if compat_os_name == 'nt':
532 if ctypes.windll.kernel32.GetConsoleWindow():
533 # c_wchar_p() might not be necessary if `message` is
534 # already of type unicode()
535 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
536 elif 'TERM' in os.environ:
537 self._write_string('\033]0;%s\007' % message, self._screen_file)
538
539 def save_console_title(self):
540 if not self.params.get('consoletitle', False):
541 return
542 if self.params.get('simulate', False):
543 return
544 if compat_os_name != 'nt' and 'TERM' in os.environ:
545 # Save the title on stack
546 self._write_string('\033[22;0t', self._screen_file)
547
548 def restore_console_title(self):
549 if not self.params.get('consoletitle', False):
550 return
551 if self.params.get('simulate', False):
552 return
553 if compat_os_name != 'nt' and 'TERM' in os.environ:
554 # Restore the title from stack
555 self._write_string('\033[23;0t', self._screen_file)
556
557 def __enter__(self):
558 self.save_console_title()
559 return self
560
561 def __exit__(self, *args):
562 self.restore_console_title()
563
564 if self.params.get('cookiefile') is not None:
565 self.cookiejar.save(ignore_discard=True, ignore_expires=True)
566
567 def trouble(self, message=None, tb=None):
568 """Determine action to take when a download problem appears.
569
570 Depending on if the downloader has been configured to ignore
571 download errors or not, this method may throw an exception or
572 not when errors are found, after printing the message.
573
574 tb, if given, is additional traceback information.
575 """
576 if message is not None:
577 self.to_stderr(message)
578 if self.params.get('verbose'):
579 if tb is None:
580 if sys.exc_info()[0]: # if .trouble has been called from an except block
581 tb = ''
582 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
583 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
584 tb += encode_compat_str(traceback.format_exc())
585 else:
586 tb_data = traceback.format_list(traceback.extract_stack())
587 tb = ''.join(tb_data)
588 self.to_stderr(tb)
589 if not self.params.get('ignoreerrors', False):
590 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
591 exc_info = sys.exc_info()[1].exc_info
592 else:
593 exc_info = sys.exc_info()
594 raise DownloadError(message, exc_info)
595 self._download_retcode = 1
596
597 def report_warning(self, message):
598 '''
599 Print the message to stderr, it will be prefixed with 'WARNING:'
600 If stderr is a tty file the 'WARNING:' will be colored
601 '''
602 if self.params.get('logger') is not None:
603 self.params['logger'].warning(message)
604 else:
605 if self.params.get('no_warnings'):
606 return
607 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
608 _msg_header = '\033[0;33mWARNING:\033[0m'
609 else:
610 _msg_header = 'WARNING:'
611 warning_message = '%s %s' % (_msg_header, message)
612 self.to_stderr(warning_message)
613
614 def report_error(self, message, tb=None):
615 '''
616 Do the same as trouble, but prefixes the message with 'ERROR:', colored
617 in red if stderr is a tty file.
618 '''
619 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
620 _msg_header = '\033[0;31mERROR:\033[0m'
621 else:
622 _msg_header = 'ERROR:'
623 error_message = '%s %s' % (_msg_header, message)
624 self.trouble(error_message, tb)
625
626 def report_file_already_downloaded(self, file_name):
627 """Report file has already been fully downloaded."""
628 try:
629 self.to_screen('[download] %s has already been downloaded' % file_name)
630 except UnicodeEncodeError:
631 self.to_screen('[download] The file has already been downloaded')
632
633 def prepare_filename(self, info_dict):
634 """Generate the output filename."""
635 try:
636 template_dict = dict(info_dict)
637
638 template_dict['epoch'] = int(time.time())
639 autonumber_size = self.params.get('autonumber_size')
640 if autonumber_size is None:
641 autonumber_size = 5
642 template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
643 if template_dict.get('resolution') is None:
644 if template_dict.get('width') and template_dict.get('height'):
645 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
646 elif template_dict.get('height'):
647 template_dict['resolution'] = '%sp' % template_dict['height']
648 elif template_dict.get('width'):
649 template_dict['resolution'] = '%dx?' % template_dict['width']
650
651 sanitize = lambda k, v: sanitize_filename(
652 compat_str(v),
653 restricted=self.params.get('restrictfilenames'),
654 is_id=(k == 'id' or k.endswith('_id')))
655 template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
656 for k, v in template_dict.items()
657 if v is not None and not isinstance(v, (list, tuple, dict)))
658 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
659
660 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
661
662 # For fields playlist_index and autonumber convert all occurrences
663 # of %(field)s to %(field)0Nd for backward compatibility
664 field_size_compat_map = {
665 'playlist_index': len(str(template_dict['n_entries'])),
666 'autonumber': autonumber_size,
667 }
668 FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
669 mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
670 if mobj:
671 outtmpl = re.sub(
672 FIELD_SIZE_COMPAT_RE,
673 r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
674 outtmpl)
675
676 # Missing numeric fields used together with integer presentation types
677 # in format specification will break the argument substitution since
678 # string 'NA' is returned for missing fields. We will patch output
679 # template for missing fields to meet string presentation type.
680 for numeric_field in self._NUMERIC_FIELDS:
681 if numeric_field not in template_dict:
682 # As of [1] format syntax is:
683 # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
684 # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
685 FORMAT_RE = r'''(?x)
686 (?<!%)
687 %
688 \({0}\) # mapping key
689 (?:[#0\-+ ]+)? # conversion flags (optional)
690 (?:\d+)? # minimum field width (optional)
691 (?:\.\d+)? # precision (optional)
692 [hlL]? # length modifier (optional)
693 [diouxXeEfFgGcrs%] # conversion type
694 '''
695 outtmpl = re.sub(
696 FORMAT_RE.format(numeric_field),
697 r'%({0})s'.format(numeric_field), outtmpl)
698
699 # expand_path translates '%%' into '%' and '$$' into '$'
700 # correspondingly that is not what we want since we need to keep
701 # '%%' intact for template dict substitution step. Working around
702 # with boundary-alike separator hack.
703 sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
704 outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
705
706 # outtmpl should be expand_path'ed before template dict substitution
707 # because meta fields may contain env variables we don't want to
708 # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
709 # title "Hello $PATH", we don't want `$PATH` to be expanded.
710 filename = expand_path(outtmpl).replace(sep, '') % template_dict
711
712 # Temporary fix for #4787
713 # 'Treat' all problem characters by passing filename through preferredencoding
714 # to workaround encoding issues with subprocess on python2 @ Windows
715 if sys.version_info < (3, 0) and sys.platform == 'win32':
716 filename = encodeFilename(filename, True).decode(preferredencoding())
717 return sanitize_path(filename)
718 except ValueError as err:
719 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
720 return None
721
722 def _match_entry(self, info_dict, incomplete):
723 """ Returns None iff the file should be downloaded """
724
725 video_title = info_dict.get('title', info_dict.get('id', 'video'))
726 if 'title' in info_dict:
727 # This can happen when we're just evaluating the playlist
728 title = info_dict['title']
729 matchtitle = self.params.get('matchtitle', False)
730 if matchtitle:
731 if not re.search(matchtitle, title, re.IGNORECASE):
732 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
733 rejecttitle = self.params.get('rejecttitle', False)
734 if rejecttitle:
735 if re.search(rejecttitle, title, re.IGNORECASE):
736 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
737 date = info_dict.get('upload_date')
738 if date is not None:
739 dateRange = self.params.get('daterange', DateRange())
740 if date not in dateRange:
741 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
742 view_count = info_dict.get('view_count')
743 if view_count is not None:
744 min_views = self.params.get('min_views')
745 if min_views is not None and view_count < min_views:
746 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
747 max_views = self.params.get('max_views')
748 if max_views is not None and view_count > max_views:
749 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
750 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
751 return 'Skipping "%s" because it is age restricted' % video_title
752 if self.in_download_archive(info_dict):
753 return '%s has already been recorded in archive' % video_title
754
755 if not incomplete:
756 match_filter = self.params.get('match_filter')
757 if match_filter is not None:
758 ret = match_filter(info_dict)
759 if ret is not None:
760 return ret
761
762 return None
763
764 @staticmethod
765 def add_extra_info(info_dict, extra_info):
766 '''Set the keys from extra_info in info dict if they are missing'''
767 for key, value in extra_info.items():
768 info_dict.setdefault(key, value)
769
770 def extract_info(self, url, download=True, ie_key=None, extra_info={},
771 process=True, force_generic_extractor=False):
772 '''
773 Returns a list with a dictionary for each video we find.
774 If 'download', also downloads the videos.
775 extra_info is a dict containing the extra values to add to each result
776 '''
777
778 if not ie_key and force_generic_extractor:
779 ie_key = 'Generic'
780
781 if ie_key:
782 ies = [self.get_info_extractor(ie_key)]
783 else:
784 ies = self._ies
785
786 for ie in ies:
787 if not ie.suitable(url):
788 continue
789
790 ie = self.get_info_extractor(ie.ie_key())
791 if not ie.working():
792 self.report_warning('The program functionality for this site has been marked as broken, '
793 'and will probably not work.')
794
795 try:
796 ie_result = ie.extract(url)
797 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
798 break
799 if isinstance(ie_result, list):
800 # Backwards compatibility: old IE result format
801 ie_result = {
802 '_type': 'compat_list',
803 'entries': ie_result,
804 }
805 self.add_default_extra_info(ie_result, ie, url)
806 if process:
807 return self.process_ie_result(ie_result, download, extra_info)
808 else:
809 return ie_result
810 except GeoRestrictedError as e:
811 msg = e.msg
812 if e.countries:
813 msg += '\nThis video is available in %s.' % ', '.join(
814 map(ISO3166Utils.short2full, e.countries))
815 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
816 self.report_error(msg)
817 break
818 except ExtractorError as e: # An error we somewhat expected
819 self.report_error(compat_str(e), e.format_traceback())
820 break
821 except MaxDownloadsReached:
822 raise
823 except Exception as e:
824 if self.params.get('ignoreerrors', False):
825 self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
826 break
827 else:
828 raise
829 else:
830 self.report_error('no suitable InfoExtractor for URL %s' % url)
831
832 def add_default_extra_info(self, ie_result, ie, url):
833 self.add_extra_info(ie_result, {
834 'extractor': ie.IE_NAME,
835 'webpage_url': url,
836 'webpage_url_basename': url_basename(url),
837 'extractor_key': ie.ie_key(),
838 })
839
840 def process_ie_result(self, ie_result, download=True, extra_info={}):
841 """
842 Take the result of the ie(may be modified) and resolve all unresolved
843 references (URLs, playlist items).
844
845 It will also download the videos if 'download'.
846 Returns the resolved ie_result.
847 """
848 result_type = ie_result.get('_type', 'video')
849
850 if result_type in ('url', 'url_transparent'):
851 ie_result['url'] = sanitize_url(ie_result['url'])
852 extract_flat = self.params.get('extract_flat', False)
853 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info)
854 or extract_flat is True):
855 if self.params.get('forcejson', False):
856 self.to_stdout(json.dumps(ie_result))
857 return ie_result
858
859 if result_type == 'video':
860 self.add_extra_info(ie_result, extra_info)
861 return self.process_video_result(ie_result, download=download)
862 elif result_type == 'url':
863 # We have to add extra_info to the results because it may be
864 # contained in a playlist
865 return self.extract_info(ie_result['url'],
866 download,
867 ie_key=ie_result.get('ie_key'),
868 extra_info=extra_info)
869 elif result_type == 'url_transparent':
870 # Use the information from the embedding page
871 info = self.extract_info(
872 ie_result['url'], ie_key=ie_result.get('ie_key'),
873 extra_info=extra_info, download=False, process=False)
874
875 # extract_info may return None when ignoreerrors is enabled and
876 # extraction failed with an error, don't crash and return early
877 # in this case
878 if not info:
879 return info
880
881 force_properties = dict(
882 (k, v) for k, v in ie_result.items() if v is not None)
883 for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
884 if f in force_properties:
885 del force_properties[f]
886 new_result = info.copy()
887 new_result.update(force_properties)
888
889 # Extracted info may not be a video result (i.e.
890 # info.get('_type', 'video') != video) but rather an url or
891 # url_transparent. In such cases outer metadata (from ie_result)
892 # should be propagated to inner one (info). For this to happen
893 # _type of info should be overridden with url_transparent. This
894 # fixes issue from https://github.com/ytdl-org/youtube-dl/pull/11163.
895 if new_result.get('_type') == 'url':
896 new_result['_type'] = 'url_transparent'
897
898 return self.process_ie_result(
899 new_result, download=download, extra_info=extra_info)
900 elif result_type in ('playlist', 'multi_video'):
901 # We process each entry in the playlist
902 playlist = ie_result.get('title') or ie_result.get('id')
903 self.to_screen('[download] Downloading playlist: %s' % playlist)
904
905 playlist_results = []
906
907 playliststart = self.params.get('playliststart', 1) - 1
908 playlistend = self.params.get('playlistend')
909 # For backwards compatibility, interpret -1 as whole list
910 if playlistend == -1:
911 playlistend = None
912
913 playlistitems_str = self.params.get('playlist_items')
914 playlistitems = None
915 if playlistitems_str is not None:
916 def iter_playlistitems(format):
917 for string_segment in format.split(','):
918 if '-' in string_segment:
919 start, end = string_segment.split('-')
920 for item in range(int(start), int(end) + 1):
921 yield int(item)
922 else:
923 yield int(string_segment)
924 playlistitems = orderedSet(iter_playlistitems(playlistitems_str))
925
926 ie_entries = ie_result['entries']
927
928 def make_playlistitems_entries(list_ie_entries):
929 num_entries = len(list_ie_entries)
930 return [
931 list_ie_entries[i - 1] for i in playlistitems
932 if -num_entries <= i - 1 < num_entries]
933
934 def report_download(num_entries):
935 self.to_screen(
936 '[%s] playlist %s: Downloading %d videos' %
937 (ie_result['extractor'], playlist, num_entries))
938
939 if isinstance(ie_entries, list):
940 n_all_entries = len(ie_entries)
941 if playlistitems:
942 entries = make_playlistitems_entries(ie_entries)
943 else:
944 entries = ie_entries[playliststart:playlistend]
945 n_entries = len(entries)
946 self.to_screen(
947 '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
948 (ie_result['extractor'], playlist, n_all_entries, n_entries))
949 elif isinstance(ie_entries, PagedList):
950 if playlistitems:
951 entries = []
952 for item in playlistitems:
953 entries.extend(ie_entries.getslice(
954 item - 1, item
955 ))
956 else:
957 entries = ie_entries.getslice(
958 playliststart, playlistend)
959 n_entries = len(entries)
960 report_download(n_entries)
961 else: # iterable
962 if playlistitems:
963 entries = make_playlistitems_entries(list(itertools.islice(
964 ie_entries, 0, max(playlistitems))))
965 else:
966 entries = list(itertools.islice(
967 ie_entries, playliststart, playlistend))
968 n_entries = len(entries)
969 report_download(n_entries)
970
971 if self.params.get('playlistreverse', False):
972 entries = entries[::-1]
973
974 if self.params.get('playlistrandom', False):
975 random.shuffle(entries)
976
977 x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
978
979 for i, entry in enumerate(entries, 1):
980 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
981 # This __x_forwarded_for_ip thing is a bit ugly but requires
982 # minimal changes
983 if x_forwarded_for:
984 entry['__x_forwarded_for_ip'] = x_forwarded_for
985 extra = {
986 'n_entries': n_entries,
987 'playlist': playlist,
988 'playlist_id': ie_result.get('id'),
989 'playlist_title': ie_result.get('title'),
990 'playlist_uploader': ie_result.get('uploader'),
991 'playlist_uploader_id': ie_result.get('uploader_id'),
992 'playlist_index': i + playliststart,
993 'extractor': ie_result['extractor'],
994 'webpage_url': ie_result['webpage_url'],
995 'webpage_url_basename': url_basename(ie_result['webpage_url']),
996 'extractor_key': ie_result['extractor_key'],
997 }
998
999 reason = self._match_entry(entry, incomplete=True)
1000 if reason is not None:
1001 self.to_screen('[download] ' + reason)
1002 continue
1003
1004 entry_result = self.process_ie_result(entry,
1005 download=download,
1006 extra_info=extra)
1007 playlist_results.append(entry_result)
1008 ie_result['entries'] = playlist_results
1009 self.to_screen('[download] Finished downloading playlist: %s' % playlist)
1010 return ie_result
1011 elif result_type == 'compat_list':
1012 self.report_warning(
1013 'Extractor %s returned a compat_list result. '
1014 'It needs to be updated.' % ie_result.get('extractor'))
1015
1016 def _fixup(r):
1017 self.add_extra_info(
1018 r,
1019 {
1020 'extractor': ie_result['extractor'],
1021 'webpage_url': ie_result['webpage_url'],
1022 'webpage_url_basename': url_basename(ie_result['webpage_url']),
1023 'extractor_key': ie_result['extractor_key'],
1024 }
1025 )
1026 return r
1027 ie_result['entries'] = [
1028 self.process_ie_result(_fixup(r), download, extra_info)
1029 for r in ie_result['entries']
1030 ]
1031 return ie_result
1032 else:
1033 raise Exception('Invalid result type: %s' % result_type)
1034
1035 def _build_format_filter(self, filter_spec):
1036 " Returns a function to filter the formats according to the filter_spec "
1037
1038 OPERATORS = {
1039 '<': operator.lt,
1040 '<=': operator.le,
1041 '>': operator.gt,
1042 '>=': operator.ge,
1043 '=': operator.eq,
1044 '!=': operator.ne,
1045 }
1046 operator_rex = re.compile(r'''(?x)\s*
1047 (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)
1048 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1049 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1050 $
1051 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
1052 m = operator_rex.search(filter_spec)
1053 if m:
1054 try:
1055 comparison_value = int(m.group('value'))
1056 except ValueError:
1057 comparison_value = parse_filesize(m.group('value'))
1058 if comparison_value is None:
1059 comparison_value = parse_filesize(m.group('value') + 'B')
1060 if comparison_value is None:
1061 raise ValueError(
1062 'Invalid value %r in format specification %r' % (
1063 m.group('value'), filter_spec))
1064 op = OPERATORS[m.group('op')]
1065
1066 if not m:
1067 STR_OPERATORS = {
1068 '=': operator.eq,
1069 '^=': lambda attr, value: attr.startswith(value),
1070 '$=': lambda attr, value: attr.endswith(value),
1071 '*=': lambda attr, value: value in attr,
1072 }
1073 str_operator_rex = re.compile(r'''(?x)
1074 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
1075 \s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?
1076 \s*(?P<value>[a-zA-Z0-9._-]+)
1077 \s*$
1078 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
1079 m = str_operator_rex.search(filter_spec)
1080 if m:
1081 comparison_value = m.group('value')
1082 str_op = STR_OPERATORS[m.group('op')]
1083 if m.group('negation'):
1084 op = lambda attr, value: not str_op(attr, value)
1085 else:
1086 op = str_op
1087
1088 if not m:
1089 raise ValueError('Invalid filter specification %r' % filter_spec)
1090
1091 def _filter(f):
1092 actual_value = f.get(m.group('key'))
1093 if actual_value is None:
1094 return m.group('none_inclusive')
1095 return op(actual_value, comparison_value)
1096 return _filter
1097
1098 def _default_format_spec(self, info_dict, download=True):
1099
1100 def can_merge():
1101 merger = FFmpegMergerPP(self)
1102 return merger.available and merger.can_merge()
1103
1104 def prefer_best():
1105 if self.params.get('simulate', False):
1106 return False
1107 if not download:
1108 return False
1109 if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
1110 return True
1111 if info_dict.get('is_live'):
1112 return True
1113 if not can_merge():
1114 return True
1115 return False
1116
1117 req_format_list = ['bestvideo+bestaudio', 'best']
1118 if prefer_best():
1119 req_format_list.reverse()
1120 return '/'.join(req_format_list)
1121
1122 def build_format_selector(self, format_spec):
1123 def syntax_error(note, start):
1124 message = (
1125 'Invalid format specification: '
1126 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
1127 return SyntaxError(message)
1128
1129 PICKFIRST = 'PICKFIRST'
1130 MERGE = 'MERGE'
1131 SINGLE = 'SINGLE'
1132 GROUP = 'GROUP'
1133 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1134
1135 def _parse_filter(tokens):
1136 filter_parts = []
1137 for type, string, start, _, _ in tokens:
1138 if type == tokenize.OP and string == ']':
1139 return ''.join(filter_parts)
1140 else:
1141 filter_parts.append(string)
1142
1143 def _remove_unused_ops(tokens):
1144 # Remove operators that we don't use and join them with the surrounding strings
1145 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1146 ALLOWED_OPS = ('/', '+', ',', '(', ')')
1147 last_string, last_start, last_end, last_line = None, None, None, None
1148 for type, string, start, end, line in tokens:
1149 if type == tokenize.OP and string == '[':
1150 if last_string:
1151 yield tokenize.NAME, last_string, last_start, last_end, last_line
1152 last_string = None
1153 yield type, string, start, end, line
1154 # everything inside brackets will be handled by _parse_filter
1155 for type, string, start, end, line in tokens:
1156 yield type, string, start, end, line
1157 if type == tokenize.OP and string == ']':
1158 break
1159 elif type == tokenize.OP and string in ALLOWED_OPS:
1160 if last_string:
1161 yield tokenize.NAME, last_string, last_start, last_end, last_line
1162 last_string = None
1163 yield type, string, start, end, line
1164 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1165 if not last_string:
1166 last_string = string
1167 last_start = start
1168 last_end = end
1169 else:
1170 last_string += string
1171 if last_string:
1172 yield tokenize.NAME, last_string, last_start, last_end, last_line
1173
1174 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1175 selectors = []
1176 current_selector = None
1177 for type, string, start, _, _ in tokens:
1178 # ENCODING is only defined in python 3.x
1179 if type == getattr(tokenize, 'ENCODING', None):
1180 continue
1181 elif type in [tokenize.NAME, tokenize.NUMBER]:
1182 current_selector = FormatSelector(SINGLE, string, [])
1183 elif type == tokenize.OP:
1184 if string == ')':
1185 if not inside_group:
1186 # ')' will be handled by the parentheses group
1187 tokens.restore_last_token()
1188 break
1189 elif inside_merge and string in ['/', ',']:
1190 tokens.restore_last_token()
1191 break
1192 elif inside_choice and string == ',':
1193 tokens.restore_last_token()
1194 break
1195 elif string == ',':
1196 if not current_selector:
1197 raise syntax_error('"," must follow a format selector', start)
1198 selectors.append(current_selector)
1199 current_selector = None
1200 elif string == '/':
1201 if not current_selector:
1202 raise syntax_error('"/" must follow a format selector', start)
1203 first_choice = current_selector
1204 second_choice = _parse_format_selection(tokens, inside_choice=True)
1205 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1206 elif string == '[':
1207 if not current_selector:
1208 current_selector = FormatSelector(SINGLE, 'best', [])
1209 format_filter = _parse_filter(tokens)
1210 current_selector.filters.append(format_filter)
1211 elif string == '(':
1212 if current_selector:
1213 raise syntax_error('Unexpected "("', start)
1214 group = _parse_format_selection(tokens, inside_group=True)
1215 current_selector = FormatSelector(GROUP, group, [])
1216 elif string == '+':
1217 video_selector = current_selector
1218 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1219 if not video_selector or not audio_selector:
1220 raise syntax_error('"+" must be between two format selectors', start)
1221 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1222 else:
1223 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1224 elif type == tokenize.ENDMARKER:
1225 break
1226 if current_selector:
1227 selectors.append(current_selector)
1228 return selectors
1229
1230 def _build_selector_function(selector):
1231 if isinstance(selector, list):
1232 fs = [_build_selector_function(s) for s in selector]
1233
1234 def selector_function(ctx):
1235 for f in fs:
1236 for format in f(ctx):
1237 yield format
1238 return selector_function
1239 elif selector.type == GROUP:
1240 selector_function = _build_selector_function(selector.selector)
1241 elif selector.type == PICKFIRST:
1242 fs = [_build_selector_function(s) for s in selector.selector]
1243
1244 def selector_function(ctx):
1245 for f in fs:
1246 picked_formats = list(f(ctx))
1247 if picked_formats:
1248 return picked_formats
1249 return []
1250 elif selector.type == SINGLE:
1251 format_spec = selector.selector
1252
1253 def selector_function(ctx):
1254 formats = list(ctx['formats'])
1255 if not formats:
1256 return
1257 if format_spec == 'all':
1258 for f in formats:
1259 yield f
1260 elif format_spec in ['best', 'worst', None]:
1261 format_idx = 0 if format_spec == 'worst' else -1
1262 audiovideo_formats = [
1263 f for f in formats
1264 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1265 if audiovideo_formats:
1266 yield audiovideo_formats[format_idx]
1267 # for extractors with incomplete formats (audio only (soundcloud)
1268 # or video only (imgur)) we will fallback to best/worst
1269 # {video,audio}-only format
1270 elif ctx['incomplete_formats']:
1271 yield formats[format_idx]
1272 elif format_spec == 'bestaudio':
1273 audio_formats = [
1274 f for f in formats
1275 if f.get('vcodec') == 'none']
1276 if audio_formats:
1277 yield audio_formats[-1]
1278 elif format_spec == 'worstaudio':
1279 audio_formats = [
1280 f for f in formats
1281 if f.get('vcodec') == 'none']
1282 if audio_formats:
1283 yield audio_formats[0]
1284 elif format_spec == 'bestvideo':
1285 video_formats = [
1286 f for f in formats
1287 if f.get('acodec') == 'none']
1288 if video_formats:
1289 yield video_formats[-1]
1290 elif format_spec == 'worstvideo':
1291 video_formats = [
1292 f for f in formats
1293 if f.get('acodec') == 'none']
1294 if video_formats:
1295 yield video_formats[0]
1296 else:
1297 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1298 if format_spec in extensions:
1299 filter_f = lambda f: f['ext'] == format_spec
1300 else:
1301 filter_f = lambda f: f['format_id'] == format_spec
1302 matches = list(filter(filter_f, formats))
1303 if matches:
1304 yield matches[-1]
1305 elif selector.type == MERGE:
1306 def _merge(formats_info):
1307 format_1, format_2 = [f['format_id'] for f in formats_info]
1308 # The first format must contain the video and the
1309 # second the audio
1310 if formats_info[0].get('vcodec') == 'none':
1311 self.report_error('The first format must '
1312 'contain the video, try using '
1313 '"-f %s+%s"' % (format_2, format_1))
1314 return
1315 # Formats must be opposite (video+audio)
1316 if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1317 self.report_error(
1318 'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1319 % (format_1, format_2))
1320 return
1321 output_ext = (
1322 formats_info[0]['ext']
1323 if self.params.get('merge_output_format') is None
1324 else self.params['merge_output_format'])
1325 return {
1326 'requested_formats': formats_info,
1327 'format': '%s+%s' % (formats_info[0].get('format'),
1328 formats_info[1].get('format')),
1329 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1330 formats_info[1].get('format_id')),
1331 'width': formats_info[0].get('width'),
1332 'height': formats_info[0].get('height'),
1333 'resolution': formats_info[0].get('resolution'),
1334 'fps': formats_info[0].get('fps'),
1335 'vcodec': formats_info[0].get('vcodec'),
1336 'vbr': formats_info[0].get('vbr'),
1337 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1338 'acodec': formats_info[1].get('acodec'),
1339 'abr': formats_info[1].get('abr'),
1340 'ext': output_ext,
1341 }
1342 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1343
1344 def selector_function(ctx):
1345 for pair in itertools.product(
1346 video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1347 yield _merge(pair)
1348
1349 filters = [self._build_format_filter(f) for f in selector.filters]
1350
1351 def final_selector(ctx):
1352 ctx_copy = copy.deepcopy(ctx)
1353 for _filter in filters:
1354 ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1355 return selector_function(ctx_copy)
1356 return final_selector
1357
1358 stream = io.BytesIO(format_spec.encode('utf-8'))
1359 try:
1360 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1361 except tokenize.TokenError:
1362 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1363
1364 class TokenIterator(object):
1365 def __init__(self, tokens):
1366 self.tokens = tokens
1367 self.counter = 0
1368
1369 def __iter__(self):
1370 return self
1371
1372 def __next__(self):
1373 if self.counter >= len(self.tokens):
1374 raise StopIteration()
1375 value = self.tokens[self.counter]
1376 self.counter += 1
1377 return value
1378
1379 next = __next__
1380
1381 def restore_last_token(self):
1382 self.counter -= 1
1383
1384 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1385 return _build_selector_function(parsed_selector)
1386
1387 def _calc_headers(self, info_dict):
1388 res = std_headers.copy()
1389
1390 add_headers = info_dict.get('http_headers')
1391 if add_headers:
1392 res.update(add_headers)
1393
1394 cookies = self._calc_cookies(info_dict)
1395 if cookies:
1396 res['Cookie'] = cookies
1397
1398 if 'X-Forwarded-For' not in res:
1399 x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
1400 if x_forwarded_for_ip:
1401 res['X-Forwarded-For'] = x_forwarded_for_ip
1402
1403 return res
1404
1405 def _calc_cookies(self, info_dict):
1406 pr = sanitized_Request(info_dict['url'])
1407 self.cookiejar.add_cookie_header(pr)
1408 return pr.get_header('Cookie')
1409
1410 def process_video_result(self, info_dict, download=True):
1411 assert info_dict.get('_type', 'video') == 'video'
1412
1413 if 'id' not in info_dict:
1414 raise ExtractorError('Missing "id" field in extractor result')
1415 if 'title' not in info_dict:
1416 raise ExtractorError('Missing "title" field in extractor result')
1417
1418 def report_force_conversion(field, field_not, conversion):
1419 self.report_warning(
1420 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1421 % (field, field_not, conversion))
1422
1423 def sanitize_string_field(info, string_field):
1424 field = info.get(string_field)
1425 if field is None or isinstance(field, compat_str):
1426 return
1427 report_force_conversion(string_field, 'a string', 'string')
1428 info[string_field] = compat_str(field)
1429
1430 def sanitize_numeric_fields(info):
1431 for numeric_field in self._NUMERIC_FIELDS:
1432 field = info.get(numeric_field)
1433 if field is None or isinstance(field, compat_numeric_types):
1434 continue
1435 report_force_conversion(numeric_field, 'numeric', 'int')
1436 info[numeric_field] = int_or_none(field)
1437
1438 sanitize_string_field(info_dict, 'id')
1439 sanitize_numeric_fields(info_dict)
1440
1441 if 'playlist' not in info_dict:
1442 # It isn't part of a playlist
1443 info_dict['playlist'] = None
1444 info_dict['playlist_index'] = None
1445
1446 thumbnails = info_dict.get('thumbnails')
1447 if thumbnails is None:
1448 thumbnail = info_dict.get('thumbnail')
1449 if thumbnail:
1450 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1451 if thumbnails:
1452 thumbnails.sort(key=lambda t: (
1453 t.get('preference') if t.get('preference') is not None else -1,
1454 t.get('width') if t.get('width') is not None else -1,
1455 t.get('height') if t.get('height') is not None else -1,
1456 t.get('id') if t.get('id') is not None else '', t.get('url')))
1457 for i, t in enumerate(thumbnails):
1458 t['url'] = sanitize_url(t['url'])
1459 if t.get('width') and t.get('height'):
1460 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1461 if t.get('id') is None:
1462 t['id'] = '%d' % i
1463
1464 if self.params.get('list_thumbnails'):
1465 self.list_thumbnails(info_dict)
1466 return
1467
1468 thumbnail = info_dict.get('thumbnail')
1469 if thumbnail:
1470 info_dict['thumbnail'] = sanitize_url(thumbnail)
1471 elif thumbnails:
1472 info_dict['thumbnail'] = thumbnails[-1]['url']
1473
1474 if 'display_id' not in info_dict and 'id' in info_dict:
1475 info_dict['display_id'] = info_dict['id']
1476
1477 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1478 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1479 # see http://bugs.python.org/issue1646728)
1480 try:
1481 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1482 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1483 except (ValueError, OverflowError, OSError):
1484 pass
1485
1486 # Auto generate title fields corresponding to the *_number fields when missing
1487 # in order to always have clean titles. This is very common for TV series.
1488 for field in ('chapter', 'season', 'episode'):
1489 if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1490 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1491
1492 for cc_kind in ('subtitles', 'automatic_captions'):
1493 cc = info_dict.get(cc_kind)
1494 if cc:
1495 for _, subtitle in cc.items():
1496 for subtitle_format in subtitle:
1497 if subtitle_format.get('url'):
1498 subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1499 if subtitle_format.get('ext') is None:
1500 subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1501
1502 automatic_captions = info_dict.get('automatic_captions')
1503 subtitles = info_dict.get('subtitles')
1504
1505 if self.params.get('listsubtitles', False):
1506 if 'automatic_captions' in info_dict:
1507 self.list_subtitles(
1508 info_dict['id'], automatic_captions, 'automatic captions')
1509 self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1510 return
1511
1512 info_dict['requested_subtitles'] = self.process_subtitles(
1513 info_dict['id'], subtitles, automatic_captions)
1514
1515 # We now pick which formats have to be downloaded
1516 if info_dict.get('formats') is None:
1517 # There's only one format available
1518 formats = [info_dict]
1519 else:
1520 formats = info_dict['formats']
1521
1522 if not formats:
1523 raise ExtractorError('No video formats found!')
1524
1525 def is_wellformed(f):
1526 url = f.get('url')
1527 if not url:
1528 self.report_warning(
1529 '"url" field is missing or empty - skipping format, '
1530 'there is an error in extractor')
1531 return False
1532 if isinstance(url, bytes):
1533 sanitize_string_field(f, 'url')
1534 return True
1535
1536 # Filter out malformed formats for better extraction robustness
1537 formats = list(filter(is_wellformed, formats))
1538
1539 formats_dict = {}
1540
1541 # We check that all the formats have the format and format_id fields
1542 for i, format in enumerate(formats):
1543 sanitize_string_field(format, 'format_id')
1544 sanitize_numeric_fields(format)
1545 format['url'] = sanitize_url(format['url'])
1546 if not format.get('format_id'):
1547 format['format_id'] = compat_str(i)
1548 else:
1549 # Sanitize format_id from characters used in format selector expression
1550 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1551 format_id = format['format_id']
1552 if format_id not in formats_dict:
1553 formats_dict[format_id] = []
1554 formats_dict[format_id].append(format)
1555
1556 # Make sure all formats have unique format_id
1557 for format_id, ambiguous_formats in formats_dict.items():
1558 if len(ambiguous_formats) > 1:
1559 for i, format in enumerate(ambiguous_formats):
1560 format['format_id'] = '%s-%d' % (format_id, i)
1561
1562 for i, format in enumerate(formats):
1563 if format.get('format') is None:
1564 format['format'] = '{id} - {res}{note}'.format(
1565 id=format['format_id'],
1566 res=self.format_resolution(format),
1567 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1568 )
1569 # Automatically determine file extension if missing
1570 if format.get('ext') is None:
1571 format['ext'] = determine_ext(format['url']).lower()
1572 # Automatically determine protocol if missing (useful for format
1573 # selection purposes)
1574 if format.get('protocol') is None:
1575 format['protocol'] = determine_protocol(format)
1576 # Add HTTP headers, so that external programs can use them from the
1577 # json output
1578 full_format_info = info_dict.copy()
1579 full_format_info.update(format)
1580 format['http_headers'] = self._calc_headers(full_format_info)
1581 # Remove private housekeeping stuff
1582 if '__x_forwarded_for_ip' in info_dict:
1583 del info_dict['__x_forwarded_for_ip']
1584
1585 # TODO Central sorting goes here
1586
1587 if formats[0] is not info_dict:
1588 # only set the 'formats' fields if the original info_dict list them
1589 # otherwise we end up with a circular reference, the first (and unique)
1590 # element in the 'formats' field in info_dict is info_dict itself,
1591 # which can't be exported to json
1592 info_dict['formats'] = formats
1593 if self.params.get('listformats'):
1594 self.list_formats(info_dict)
1595 return
1596
1597 req_format = self.params.get('format')
1598 if req_format is None:
1599 req_format = self._default_format_spec(info_dict, download=download)
1600 if self.params.get('verbose'):
1601 self.to_stdout('[debug] Default format spec: %s' % req_format)
1602
1603 format_selector = self.build_format_selector(req_format)
1604
1605 # While in format selection we may need to have an access to the original
1606 # format set in order to calculate some metrics or do some processing.
1607 # For now we need to be able to guess whether original formats provided
1608 # by extractor are incomplete or not (i.e. whether extractor provides only
1609 # video-only or audio-only formats) for proper formats selection for
1610 # extractors with such incomplete formats (see
1611 # https://github.com/ytdl-org/youtube-dl/pull/5556).
1612 # Since formats may be filtered during format selection and may not match
1613 # the original formats the results may be incorrect. Thus original formats
1614 # or pre-calculated metrics should be passed to format selection routines
1615 # as well.
1616 # We will pass a context object containing all necessary additional data
1617 # instead of just formats.
1618 # This fixes incorrect format selection issue (see
1619 # https://github.com/ytdl-org/youtube-dl/issues/10083).
1620 incomplete_formats = (
1621 # All formats are video-only or
1622 all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats)
1623 # all formats are audio-only
1624 or all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1625
1626 ctx = {
1627 'formats': formats,
1628 'incomplete_formats': incomplete_formats,
1629 }
1630
1631 formats_to_download = list(format_selector(ctx))
1632 if not formats_to_download:
1633 raise ExtractorError('requested format not available',
1634 expected=True)
1635
1636 if download:
1637 if len(formats_to_download) > 1:
1638 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1639 for format in formats_to_download:
1640 new_info = dict(info_dict)
1641 new_info.update(format)
1642 self.process_info(new_info)
1643 # We update the info dict with the best quality format (backwards compatibility)
1644 info_dict.update(formats_to_download[-1])
1645 return info_dict
1646
1647 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1648 """Select the requested subtitles and their format"""
1649 available_subs = {}
1650 if normal_subtitles and self.params.get('writesubtitles'):
1651 available_subs.update(normal_subtitles)
1652 if automatic_captions and self.params.get('writeautomaticsub'):
1653 for lang, cap_info in automatic_captions.items():
1654 if lang not in available_subs:
1655 available_subs[lang] = cap_info
1656
1657 if (not self.params.get('writesubtitles') and not
1658 self.params.get('writeautomaticsub') or not
1659 available_subs):
1660 return None
1661
1662 if self.params.get('allsubtitles', False):
1663 requested_langs = available_subs.keys()
1664 else:
1665 if self.params.get('subtitleslangs', False):
1666 requested_langs = self.params.get('subtitleslangs')
1667 elif 'en' in available_subs:
1668 requested_langs = ['en']
1669 else:
1670 requested_langs = [list(available_subs.keys())[0]]
1671
1672 formats_query = self.params.get('subtitlesformat', 'best')
1673 formats_preference = formats_query.split('/') if formats_query else []
1674 subs = {}
1675 for lang in requested_langs:
1676 formats = available_subs.get(lang)
1677 if formats is None:
1678 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1679 continue
1680 for ext in formats_preference:
1681 if ext == 'best':
1682 f = formats[-1]
1683 break
1684 matches = list(filter(lambda f: f['ext'] == ext, formats))
1685 if matches:
1686 f = matches[-1]
1687 break
1688 else:
1689 f = formats[-1]
1690 self.report_warning(
1691 'No subtitle format found matching "%s" for language %s, '
1692 'using %s' % (formats_query, lang, f['ext']))
1693 subs[lang] = f
1694 return subs
1695
1696 def process_info(self, info_dict):
1697 """Process a single resolved IE result."""
1698
1699 assert info_dict.get('_type', 'video') == 'video'
1700
1701 max_downloads = self.params.get('max_downloads')
1702 if max_downloads is not None:
1703 if self._num_downloads >= int(max_downloads):
1704 raise MaxDownloadsReached()
1705
1706 info_dict['fulltitle'] = info_dict['title']
1707 if len(info_dict['title']) > 200:
1708 info_dict['title'] = info_dict['title'][:197] + '...'
1709
1710 if 'format' not in info_dict:
1711 info_dict['format'] = info_dict['ext']
1712
1713 reason = self._match_entry(info_dict, incomplete=False)
1714 if reason is not None:
1715 self.to_screen('[download] ' + reason)
1716 return
1717
1718 self._num_downloads += 1
1719
1720 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1721
1722 # Forced printings
1723 if self.params.get('forcetitle', False):
1724 self.to_stdout(info_dict['fulltitle'])
1725 if self.params.get('forceid', False):
1726 self.to_stdout(info_dict['id'])
1727 if self.params.get('forceurl', False):
1728 if info_dict.get('requested_formats') is not None:
1729 for f in info_dict['requested_formats']:
1730 self.to_stdout(f['url'] + f.get('play_path', ''))
1731 else:
1732 # For RTMP URLs, also include the playpath
1733 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1734 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1735 self.to_stdout(info_dict['thumbnail'])
1736 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1737 self.to_stdout(info_dict['description'])
1738 if self.params.get('forcefilename', False) and filename is not None:
1739 self.to_stdout(filename)
1740 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1741 self.to_stdout(formatSeconds(info_dict['duration']))
1742 if self.params.get('forceformat', False):
1743 self.to_stdout(info_dict['format'])
1744 if self.params.get('forcejson', False):
1745 self.to_stdout(json.dumps(info_dict))
1746
1747 # Do nothing else if in simulate mode
1748 if self.params.get('simulate', False):
1749 return
1750
1751 if filename is None:
1752 return
1753
1754 def ensure_dir_exists(path):
1755 try:
1756 dn = os.path.dirname(path)
1757 if dn and not os.path.exists(dn):
1758 os.makedirs(dn)
1759 return True
1760 except (OSError, IOError) as err:
1761 self.report_error('unable to create directory ' + error_to_compat_str(err))
1762 return False
1763
1764 if not ensure_dir_exists(sanitize_path(encodeFilename(filename))):
1765 return
1766
1767 if self.params.get('writedescription', False):
1768 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1769 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1770 self.to_screen('[info] Video description is already present')
1771 elif info_dict.get('description') is None:
1772 self.report_warning('There\'s no description to write.')
1773 else:
1774 try:
1775 self.to_screen('[info] Writing video description to: ' + descfn)
1776 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1777 descfile.write(info_dict['description'])
1778 except (OSError, IOError):
1779 self.report_error('Cannot write description file ' + descfn)
1780 return
1781
1782 if self.params.get('writeannotations', False):
1783 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1784 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1785 self.to_screen('[info] Video annotations are already present')
1786 else:
1787 try:
1788 self.to_screen('[info] Writing video annotations to: ' + annofn)
1789 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1790 annofile.write(info_dict['annotations'])
1791 except (KeyError, TypeError):
1792 self.report_warning('There are no annotations to write.')
1793 except (OSError, IOError):
1794 self.report_error('Cannot write annotations file: ' + annofn)
1795 return
1796
1797 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1798 self.params.get('writeautomaticsub')])
1799
1800 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1801 # subtitles download errors are already managed as troubles in relevant IE
1802 # that way it will silently go on when used with unsupporting IE
1803 subtitles = info_dict['requested_subtitles']
1804 ie = self.get_info_extractor(info_dict['extractor_key'])
1805 for sub_lang, sub_info in subtitles.items():
1806 sub_format = sub_info['ext']
1807 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1808 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1809 self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
1810 else:
1811 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1812 if sub_info.get('data') is not None:
1813 try:
1814 # Use newline='' to prevent conversion of newline characters
1815 # See https://github.com/ytdl-org/youtube-dl/issues/10268
1816 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1817 subfile.write(sub_info['data'])
1818 except (OSError, IOError):
1819 self.report_error('Cannot write subtitles file ' + sub_filename)
1820 return
1821 else:
1822 try:
1823 sub_data = ie._request_webpage(
1824 sub_info['url'], info_dict['id'], note=False).read()
1825 with io.open(encodeFilename(sub_filename), 'wb') as subfile:
1826 subfile.write(sub_data)
1827 except (ExtractorError, IOError, OSError, ValueError) as err:
1828 self.report_warning('Unable to download subtitle for "%s": %s' %
1829 (sub_lang, error_to_compat_str(err)))
1830 continue
1831
1832 if self.params.get('writeinfojson', False):
1833 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1834 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1835 self.to_screen('[info] Video description metadata is already present')
1836 else:
1837 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1838 try:
1839 write_json_file(self.filter_requested_info(info_dict), infofn)
1840 except (OSError, IOError):
1841 self.report_error('Cannot write metadata to JSON file ' + infofn)
1842 return
1843
1844 self._write_thumbnails(info_dict, filename)
1845
1846 if not self.params.get('skip_download', False):
1847 try:
1848 def dl(name, info):
1849 fd = get_suitable_downloader(info, self.params)(self, self.params)
1850 for ph in self._progress_hooks:
1851 fd.add_progress_hook(ph)
1852 if self.params.get('verbose'):
1853 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1854 return fd.download(name, info)
1855
1856 if info_dict.get('requested_formats') is not None:
1857 downloaded = []
1858 success = True
1859 merger = FFmpegMergerPP(self)
1860 if not merger.available:
1861 postprocessors = []
1862 self.report_warning('You have requested multiple '
1863 'formats but ffmpeg or avconv are not installed.'
1864 ' The formats won\'t be merged.')
1865 else:
1866 postprocessors = [merger]
1867
1868 def compatible_formats(formats):
1869 video, audio = formats
1870 # Check extension
1871 video_ext, audio_ext = video.get('ext'), audio.get('ext')
1872 if video_ext and audio_ext:
1873 COMPATIBLE_EXTS = (
1874 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1875 ('webm')
1876 )
1877 for exts in COMPATIBLE_EXTS:
1878 if video_ext in exts and audio_ext in exts:
1879 return True
1880 # TODO: Check acodec/vcodec
1881 return False
1882
1883 filename_real_ext = os.path.splitext(filename)[1][1:]
1884 filename_wo_ext = (
1885 os.path.splitext(filename)[0]
1886 if filename_real_ext == info_dict['ext']
1887 else filename)
1888 requested_formats = info_dict['requested_formats']
1889 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1890 info_dict['ext'] = 'mkv'
1891 self.report_warning(
1892 'Requested formats are incompatible for merge and will be merged into mkv.')
1893 # Ensure filename always has a correct extension for successful merge
1894 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1895 if os.path.exists(encodeFilename(filename)):
1896 self.to_screen(
1897 '[download] %s has already been downloaded and '
1898 'merged' % filename)
1899 else:
1900 for f in requested_formats:
1901 new_info = dict(info_dict)
1902 new_info.update(f)
1903 fname = prepend_extension(
1904 self.prepare_filename(new_info),
1905 'f%s' % f['format_id'], new_info['ext'])
1906 if not ensure_dir_exists(fname):
1907 return
1908 downloaded.append(fname)
1909 partial_success = dl(fname, new_info)
1910 success = success and partial_success
1911 info_dict['__postprocessors'] = postprocessors
1912 info_dict['__files_to_merge'] = downloaded
1913 else:
1914 # Just a single file
1915 success = dl(filename, info_dict)
1916 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1917 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1918 return
1919 except (OSError, IOError) as err:
1920 raise UnavailableVideoError(err)
1921 except (ContentTooShortError, ) as err:
1922 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1923 return
1924
1925 if success and filename != '-':
1926 # Fixup content
1927 fixup_policy = self.params.get('fixup')
1928 if fixup_policy is None:
1929 fixup_policy = 'detect_or_warn'
1930
1931 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1932
1933 stretched_ratio = info_dict.get('stretched_ratio')
1934 if stretched_ratio is not None and stretched_ratio != 1:
1935 if fixup_policy == 'warn':
1936 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1937 info_dict['id'], stretched_ratio))
1938 elif fixup_policy == 'detect_or_warn':
1939 stretched_pp = FFmpegFixupStretchedPP(self)
1940 if stretched_pp.available:
1941 info_dict.setdefault('__postprocessors', [])
1942 info_dict['__postprocessors'].append(stretched_pp)
1943 else:
1944 self.report_warning(
1945 '%s: Non-uniform pixel ratio (%s). %s'
1946 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1947 else:
1948 assert fixup_policy in ('ignore', 'never')
1949
1950 if (info_dict.get('requested_formats') is None
1951 and info_dict.get('container') == 'm4a_dash'):
1952 if fixup_policy == 'warn':
1953 self.report_warning(
1954 '%s: writing DASH m4a. '
1955 'Only some players support this container.'
1956 % info_dict['id'])
1957 elif fixup_policy == 'detect_or_warn':
1958 fixup_pp = FFmpegFixupM4aPP(self)
1959 if fixup_pp.available:
1960 info_dict.setdefault('__postprocessors', [])
1961 info_dict['__postprocessors'].append(fixup_pp)
1962 else:
1963 self.report_warning(
1964 '%s: writing DASH m4a. '
1965 'Only some players support this container. %s'
1966 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1967 else:
1968 assert fixup_policy in ('ignore', 'never')
1969
1970 if (info_dict.get('protocol') == 'm3u8_native'
1971 or info_dict.get('protocol') == 'm3u8'
1972 and self.params.get('hls_prefer_native')):
1973 if fixup_policy == 'warn':
1974 self.report_warning('%s: malformed AAC bitstream detected.' % (
1975 info_dict['id']))
1976 elif fixup_policy == 'detect_or_warn':
1977 fixup_pp = FFmpegFixupM3u8PP(self)
1978 if fixup_pp.available:
1979 info_dict.setdefault('__postprocessors', [])
1980 info_dict['__postprocessors'].append(fixup_pp)
1981 else:
1982 self.report_warning(
1983 '%s: malformed AAC bitstream detected. %s'
1984 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1985 else:
1986 assert fixup_policy in ('ignore', 'never')
1987
1988 try:
1989 self.post_process(filename, info_dict)
1990 except (PostProcessingError) as err:
1991 self.report_error('postprocessing: %s' % str(err))
1992 return
1993 self.record_download_archive(info_dict)
1994
1995 def download(self, url_list):
1996 """Download a given list of URLs."""
1997 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1998 if (len(url_list) > 1
1999 and outtmpl != '-'
2000 and '%' not in outtmpl
2001 and self.params.get('max_downloads') != 1):
2002 raise SameFileError(outtmpl)
2003
2004 for url in url_list:
2005 try:
2006 # It also downloads the videos
2007 res = self.extract_info(
2008 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
2009 except UnavailableVideoError:
2010 self.report_error('unable to download video')
2011 except MaxDownloadsReached:
2012 self.to_screen('[info] Maximum number of downloaded files reached.')
2013 raise
2014 else:
2015 if self.params.get('dump_single_json', False):
2016 self.to_stdout(json.dumps(res))
2017
2018 return self._download_retcode
2019
2020 def download_with_info_file(self, info_filename):
2021 with contextlib.closing(fileinput.FileInput(
2022 [info_filename], mode='r',
2023 openhook=fileinput.hook_encoded('utf-8'))) as f:
2024 # FileInput doesn't have a read method, we can't call json.load
2025 info = self.filter_requested_info(json.loads('\n'.join(f)))
2026 try:
2027 self.process_ie_result(info, download=True)
2028 except DownloadError:
2029 webpage_url = info.get('webpage_url')
2030 if webpage_url is not None:
2031 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
2032 return self.download([webpage_url])
2033 else:
2034 raise
2035 return self._download_retcode
2036
2037 @staticmethod
2038 def filter_requested_info(info_dict):
2039 return dict(
2040 (k, v) for k, v in info_dict.items()
2041 if k not in ['requested_formats', 'requested_subtitles'])
2042
2043 def post_process(self, filename, ie_info):
2044 """Run all the postprocessors on the given file."""
2045 info = dict(ie_info)
2046 info['filepath'] = filename
2047 pps_chain = []
2048 if ie_info.get('__postprocessors') is not None:
2049 pps_chain.extend(ie_info['__postprocessors'])
2050 pps_chain.extend(self._pps)
2051 for pp in pps_chain:
2052 files_to_delete = []
2053 try:
2054 files_to_delete, info = pp.run(info)
2055 except PostProcessingError as e:
2056 self.report_error(e.msg)
2057 if files_to_delete and not self.params.get('keepvideo', False):
2058 for old_filename in files_to_delete:
2059 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
2060 try:
2061 os.remove(encodeFilename(old_filename))
2062 except (IOError, OSError):
2063 self.report_warning('Unable to remove downloaded original file')
2064
2065 def _make_archive_id(self, info_dict):
2066 video_id = info_dict.get('id')
2067 if not video_id:
2068 return
2069 # Future-proof against any change in case
2070 # and backwards compatibility with prior versions
2071 extractor = info_dict.get('extractor_key') or info_dict.get('ie_key') # key in a playlist
2072 if extractor is None:
2073 url = str_or_none(info_dict.get('url'))
2074 if not url:
2075 return
2076 # Try to find matching extractor for the URL and take its ie_key
2077 for ie in self._ies:
2078 if ie.suitable(url):
2079 extractor = ie.ie_key()
2080 break
2081 else:
2082 return
2083 return extractor.lower() + ' ' + video_id
2084
2085 def in_download_archive(self, info_dict):
2086 fn = self.params.get('download_archive')
2087 if fn is None:
2088 return False
2089
2090 vid_id = self._make_archive_id(info_dict)
2091 if not vid_id:
2092 return False # Incomplete video information
2093
2094 try:
2095 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
2096 for line in archive_file:
2097 if line.strip() == vid_id:
2098 return True
2099 except IOError as ioe:
2100 if ioe.errno != errno.ENOENT:
2101 raise
2102 return False
2103
2104 def record_download_archive(self, info_dict):
2105 fn = self.params.get('download_archive')
2106 if fn is None:
2107 return
2108 vid_id = self._make_archive_id(info_dict)
2109 assert vid_id
2110 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
2111 archive_file.write(vid_id + '\n')
2112
2113 @staticmethod
2114 def format_resolution(format, default='unknown'):
2115 if format.get('vcodec') == 'none':
2116 return 'audio only'
2117 if format.get('resolution') is not None:
2118 return format['resolution']
2119 if format.get('height') is not None:
2120 if format.get('width') is not None:
2121 res = '%sx%s' % (format['width'], format['height'])
2122 else:
2123 res = '%sp' % format['height']
2124 elif format.get('width') is not None:
2125 res = '%dx?' % format['width']
2126 else:
2127 res = default
2128 return res
2129
2130 def _format_note(self, fdict):
2131 res = ''
2132 if fdict.get('ext') in ['f4f', 'f4m']:
2133 res += '(unsupported) '
2134 if fdict.get('language'):
2135 if res:
2136 res += ' '
2137 res += '[%s] ' % fdict['language']
2138 if fdict.get('format_note') is not None:
2139 res += fdict['format_note'] + ' '
2140 if fdict.get('tbr') is not None:
2141 res += '%4dk ' % fdict['tbr']
2142 if fdict.get('container') is not None:
2143 if res:
2144 res += ', '
2145 res += '%s container' % fdict['container']
2146 if (fdict.get('vcodec') is not None
2147 and fdict.get('vcodec') != 'none'):
2148 if res:
2149 res += ', '
2150 res += fdict['vcodec']
2151 if fdict.get('vbr') is not None:
2152 res += '@'
2153 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
2154 res += 'video@'
2155 if fdict.get('vbr') is not None:
2156 res += '%4dk' % fdict['vbr']
2157 if fdict.get('fps') is not None:
2158 if res:
2159 res += ', '
2160 res += '%sfps' % fdict['fps']
2161 if fdict.get('acodec') is not None:
2162 if res:
2163 res += ', '
2164 if fdict['acodec'] == 'none':
2165 res += 'video only'
2166 else:
2167 res += '%-5s' % fdict['acodec']
2168 elif fdict.get('abr') is not None:
2169 if res:
2170 res += ', '
2171 res += 'audio'
2172 if fdict.get('abr') is not None:
2173 res += '@%3dk' % fdict['abr']
2174 if fdict.get('asr') is not None:
2175 res += ' (%5dHz)' % fdict['asr']
2176 if fdict.get('filesize') is not None:
2177 if res:
2178 res += ', '
2179 res += format_bytes(fdict['filesize'])
2180 elif fdict.get('filesize_approx') is not None:
2181 if res:
2182 res += ', '
2183 res += '~' + format_bytes(fdict['filesize_approx'])
2184 return res
2185
2186 def list_formats(self, info_dict):
2187 formats = info_dict.get('formats', [info_dict])
2188 table = [
2189 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
2190 for f in formats
2191 if f.get('preference') is None or f['preference'] >= -1000]
2192 if len(formats) > 1:
2193 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2194
2195 header_line = ['format code', 'extension', 'resolution', 'note']
2196 self.to_screen(
2197 '[info] Available formats for %s:\n%s' %
2198 (info_dict['id'], render_table(header_line, table)))
2199
2200 def list_thumbnails(self, info_dict):
2201 thumbnails = info_dict.get('thumbnails')
2202 if not thumbnails:
2203 self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
2204 return
2205
2206 self.to_screen(
2207 '[info] Thumbnails for %s:' % info_dict['id'])
2208 self.to_screen(render_table(
2209 ['ID', 'width', 'height', 'URL'],
2210 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
2211
2212 def list_subtitles(self, video_id, subtitles, name='subtitles'):
2213 if not subtitles:
2214 self.to_screen('%s has no %s' % (video_id, name))
2215 return
2216 self.to_screen(
2217 'Available %s for %s:' % (name, video_id))
2218 self.to_screen(render_table(
2219 ['Language', 'formats'],
2220 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2221 for lang, formats in subtitles.items()]))
2222
2223 def urlopen(self, req):
2224 """ Start an HTTP download """
2225 if isinstance(req, compat_basestring):
2226 req = sanitized_Request(req)
2227 return self._opener.open(req, timeout=self._socket_timeout)
2228
2229 def print_debug_header(self):
2230 if not self.params.get('verbose'):
2231 return
2232
2233 if type('') is not compat_str:
2234 # Python 2.6 on SLES11 SP1 (https://github.com/ytdl-org/youtube-dl/issues/3326)
2235 self.report_warning(
2236 'Your Python is broken! Update to a newer and supported version')
2237
2238 stdout_encoding = getattr(
2239 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2240 encoding_str = (
2241 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2242 locale.getpreferredencoding(),
2243 sys.getfilesystemencoding(),
2244 stdout_encoding,
2245 self.get_encoding()))
2246 write_string(encoding_str, encoding=None)
2247
2248 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2249 if _LAZY_LOADER:
2250 self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2251 try:
2252 sp = subprocess.Popen(
2253 ['git', 'rev-parse', '--short', 'HEAD'],
2254 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2255 cwd=os.path.dirname(os.path.abspath(__file__)))
2256 out, err = sp.communicate()
2257 out = out.decode().strip()
2258 if re.match('[0-9a-f]+', out):
2259 self._write_string('[debug] Git HEAD: ' + out + '\n')
2260 except Exception:
2261 try:
2262 sys.exc_clear()
2263 except Exception:
2264 pass
2265
2266 def python_implementation():
2267 impl_name = platform.python_implementation()
2268 if impl_name == 'PyPy' and hasattr(sys, 'pypy_version_info'):
2269 return impl_name + ' version %d.%d.%d' % sys.pypy_version_info[:3]
2270 return impl_name
2271
2272 self._write_string('[debug] Python version %s (%s) - %s\n' % (
2273 platform.python_version(), python_implementation(),
2274 platform_name()))
2275
2276 exe_versions = FFmpegPostProcessor.get_versions(self)
2277 exe_versions['rtmpdump'] = rtmpdump_version()
2278 exe_versions['phantomjs'] = PhantomJSwrapper._version()
2279 exe_str = ', '.join(
2280 '%s %s' % (exe, v)
2281 for exe, v in sorted(exe_versions.items())
2282 if v
2283 )
2284 if not exe_str:
2285 exe_str = 'none'
2286 self._write_string('[debug] exe versions: %s\n' % exe_str)
2287
2288 proxy_map = {}
2289 for handler in self._opener.handlers:
2290 if hasattr(handler, 'proxies'):
2291 proxy_map.update(handler.proxies)
2292 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2293
2294 if self.params.get('call_home', False):
2295 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2296 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2297 latest_version = self.urlopen(
2298 'https://yt-dl.org/latest/version').read().decode('utf-8')
2299 if version_tuple(latest_version) > version_tuple(__version__):
2300 self.report_warning(
2301 'You are using an outdated version (newest version: %s)! '
2302 'See https://yt-dl.org/update if you need help updating.' %
2303 latest_version)
2304
2305 def _setup_opener(self):
2306 timeout_val = self.params.get('socket_timeout')
2307 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2308
2309 opts_cookiefile = self.params.get('cookiefile')
2310 opts_proxy = self.params.get('proxy')
2311
2312 if opts_cookiefile is None:
2313 self.cookiejar = compat_cookiejar.CookieJar()
2314 else:
2315 opts_cookiefile = expand_path(opts_cookiefile)
2316 self.cookiejar = YoutubeDLCookieJar(opts_cookiefile)
2317 if os.access(opts_cookiefile, os.R_OK):
2318 self.cookiejar.load(ignore_discard=True, ignore_expires=True)
2319
2320 cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2321 if opts_proxy is not None:
2322 if opts_proxy == '':
2323 proxies = {}
2324 else:
2325 proxies = {'http': opts_proxy, 'https': opts_proxy}
2326 else:
2327 proxies = compat_urllib_request.getproxies()
2328 # Set HTTPS proxy to HTTP one if given (https://github.com/ytdl-org/youtube-dl/issues/805)
2329 if 'http' in proxies and 'https' not in proxies:
2330 proxies['https'] = proxies['http']
2331 proxy_handler = PerRequestProxyHandler(proxies)
2332
2333 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2334 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2335 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2336 data_handler = compat_urllib_request_DataHandler()
2337
2338 # When passing our own FileHandler instance, build_opener won't add the
2339 # default FileHandler and allows us to disable the file protocol, which
2340 # can be used for malicious purposes (see
2341 # https://github.com/ytdl-org/youtube-dl/issues/8227)
2342 file_handler = compat_urllib_request.FileHandler()
2343
2344 def file_open(*args, **kwargs):
2345 raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2346 file_handler.file_open = file_open
2347
2348 opener = compat_urllib_request.build_opener(
2349 proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2350
2351 # Delete the default user-agent header, which would otherwise apply in
2352 # cases where our custom HTTP handler doesn't come into play
2353 # (See https://github.com/ytdl-org/youtube-dl/issues/1309 for details)
2354 opener.addheaders = []
2355 self._opener = opener
2356
2357 def encode(self, s):
2358 if isinstance(s, bytes):
2359 return s # Already encoded
2360
2361 try:
2362 return s.encode(self.get_encoding())
2363 except UnicodeEncodeError as err:
2364 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2365 raise
2366
2367 def get_encoding(self):
2368 encoding = self.params.get('encoding')
2369 if encoding is None:
2370 encoding = preferredencoding()
2371 return encoding
2372
2373 def _write_thumbnails(self, info_dict, filename):
2374 if self.params.get('writethumbnail', False):
2375 thumbnails = info_dict.get('thumbnails')
2376 if thumbnails:
2377 thumbnails = [thumbnails[-1]]
2378 elif self.params.get('write_all_thumbnails', False):
2379 thumbnails = info_dict.get('thumbnails')
2380 else:
2381 return
2382
2383 if not thumbnails:
2384 # No thumbnails present, so return immediately
2385 return
2386
2387 for t in thumbnails:
2388 thumb_ext = determine_ext(t['url'], 'jpg')
2389 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2390 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2391 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2392
2393 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2394 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2395 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2396 else:
2397 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2398 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2399 try:
2400 uf = self.urlopen(t['url'])
2401 with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2402 shutil.copyfileobj(uf, thumbf)
2403 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2404 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2405 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2406 self.report_warning('Unable to download thumbnail "%s": %s' %
2407 (t['url'], error_to_compat_str(err)))