]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py
5036289b062f9ee05ab5c872c76ab38babcb4a54
[youtubedl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import absolute_import, unicode_literals
5
6 import collections
7 import contextlib
8 import datetime
9 import errno
10 import fileinput
11 import io
12 import itertools
13 import json
14 import locale
15 import operator
16 import os
17 import platform
18 import re
19 import shutil
20 import subprocess
21 import socket
22 import sys
23 import time
24 import tokenize
25 import traceback
26
27 from .compat import (
28 compat_basestring,
29 compat_cookiejar,
30 compat_expanduser,
31 compat_get_terminal_size,
32 compat_http_client,
33 compat_kwargs,
34 compat_os_name,
35 compat_str,
36 compat_tokenize_tokenize,
37 compat_urllib_error,
38 compat_urllib_request,
39 compat_urllib_request_DataHandler,
40 )
41 from .utils import (
42 age_restricted,
43 args_to_str,
44 ContentTooShortError,
45 date_from_str,
46 DateRange,
47 DEFAULT_OUTTMPL,
48 determine_ext,
49 determine_protocol,
50 DownloadError,
51 encode_compat_str,
52 encodeFilename,
53 error_to_compat_str,
54 ExtractorError,
55 format_bytes,
56 formatSeconds,
57 locked_file,
58 make_HTTPS_handler,
59 MaxDownloadsReached,
60 PagedList,
61 parse_filesize,
62 PerRequestProxyHandler,
63 platform_name,
64 PostProcessingError,
65 preferredencoding,
66 prepend_extension,
67 register_socks_protocols,
68 render_table,
69 replace_extension,
70 SameFileError,
71 sanitize_filename,
72 sanitize_path,
73 sanitize_url,
74 sanitized_Request,
75 std_headers,
76 subtitles_filename,
77 UnavailableVideoError,
78 url_basename,
79 version_tuple,
80 write_json_file,
81 write_string,
82 YoutubeDLCookieProcessor,
83 YoutubeDLHandler,
84 )
85 from .cache import Cache
86 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
87 from .downloader import get_suitable_downloader
88 from .downloader.rtmp import rtmpdump_version
89 from .postprocessor import (
90 FFmpegFixupM3u8PP,
91 FFmpegFixupM4aPP,
92 FFmpegFixupStretchedPP,
93 FFmpegMergerPP,
94 FFmpegPostProcessor,
95 get_postprocessor,
96 )
97 from .version import __version__
98
99 if compat_os_name == 'nt':
100 import ctypes
101
102
103 class YoutubeDL(object):
104 """YoutubeDL class.
105
106 YoutubeDL objects are the ones responsible of downloading the
107 actual video file and writing it to disk if the user has requested
108 it, among some other tasks. In most cases there should be one per
109 program. As, given a video URL, the downloader doesn't know how to
110 extract all the needed information, task that InfoExtractors do, it
111 has to pass the URL to one of them.
112
113 For this, YoutubeDL objects have a method that allows
114 InfoExtractors to be registered in a given order. When it is passed
115 a URL, the YoutubeDL object handles it to the first InfoExtractor it
116 finds that reports being able to handle it. The InfoExtractor extracts
117 all the information about the video or videos the URL refers to, and
118 YoutubeDL process the extracted information, possibly using a File
119 Downloader to download the video.
120
121 YoutubeDL objects accept a lot of parameters. In order not to saturate
122 the object constructor with arguments, it receives a dictionary of
123 options instead. These options are available through the params
124 attribute for the InfoExtractors to use. The YoutubeDL also
125 registers itself as the downloader in charge for the InfoExtractors
126 that are added to it, so this is a "mutual registration".
127
128 Available options:
129
130 username: Username for authentication purposes.
131 password: Password for authentication purposes.
132 videopassword: Password for accessing a video.
133 usenetrc: Use netrc for authentication instead.
134 verbose: Print additional info to stdout.
135 quiet: Do not print messages to stdout.
136 no_warnings: Do not print out anything for warnings.
137 forceurl: Force printing final URL.
138 forcetitle: Force printing title.
139 forceid: Force printing ID.
140 forcethumbnail: Force printing thumbnail URL.
141 forcedescription: Force printing description.
142 forcefilename: Force printing final filename.
143 forceduration: Force printing duration.
144 forcejson: Force printing info_dict as JSON.
145 dump_single_json: Force printing the info_dict of the whole playlist
146 (or video) as a single JSON line.
147 simulate: Do not download the video files.
148 format: Video format code. See options.py for more information.
149 outtmpl: Template for output names.
150 restrictfilenames: Do not allow "&" and spaces in file names
151 ignoreerrors: Do not stop on download errors.
152 force_generic_extractor: Force downloader to use the generic extractor
153 nooverwrites: Prevent overwriting files.
154 playliststart: Playlist item to start at.
155 playlistend: Playlist item to end at.
156 playlist_items: Specific indices of playlist to download.
157 playlistreverse: Download playlist items in reverse order.
158 matchtitle: Download only matching titles.
159 rejecttitle: Reject downloads for matching titles.
160 logger: Log messages to a logging.Logger instance.
161 logtostderr: Log messages to stderr instead of stdout.
162 writedescription: Write the video description to a .description file
163 writeinfojson: Write the video description to a .info.json file
164 writeannotations: Write the video annotations to a .annotations.xml file
165 writethumbnail: Write the thumbnail image to a file
166 write_all_thumbnails: Write all thumbnail formats to files
167 writesubtitles: Write the video subtitles to a file
168 writeautomaticsub: Write the automatically generated subtitles to a file
169 allsubtitles: Downloads all the subtitles of the video
170 (requires writesubtitles or writeautomaticsub)
171 listsubtitles: Lists all available subtitles for the video
172 subtitlesformat: The format code for subtitles
173 subtitleslangs: List of languages of the subtitles to download
174 keepvideo: Keep the video file after post-processing
175 daterange: A DateRange object, download only if the upload_date is in the range.
176 skip_download: Skip the actual download of the video file
177 cachedir: Location of the cache files in the filesystem.
178 False to disable filesystem cache.
179 noplaylist: Download single video instead of a playlist if in doubt.
180 age_limit: An integer representing the user's age in years.
181 Unsuitable videos for the given age are skipped.
182 min_views: An integer representing the minimum view count the video
183 must have in order to not be skipped.
184 Videos without view count information are always
185 downloaded. None for no limit.
186 max_views: An integer representing the maximum view count.
187 Videos that are more popular than that are not
188 downloaded.
189 Videos without view count information are always
190 downloaded. None for no limit.
191 download_archive: File name of a file where all downloads are recorded.
192 Videos already present in the file are not downloaded
193 again.
194 cookiefile: File name where cookies should be read from and dumped to.
195 nocheckcertificate:Do not verify SSL certificates
196 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
197 At the moment, this is only supported by YouTube.
198 proxy: URL of the proxy server to use
199 cn_verification_proxy: URL of the proxy to use for IP address verification
200 on Chinese sites. (Experimental)
201 socket_timeout: Time to wait for unresponsive hosts, in seconds
202 bidi_workaround: Work around buggy terminals without bidirectional text
203 support, using fridibi
204 debug_printtraffic:Print out sent and received HTTP traffic
205 include_ads: Download ads as well
206 default_search: Prepend this string if an input url is not valid.
207 'auto' for elaborate guessing
208 encoding: Use this encoding instead of the system-specified.
209 extract_flat: Do not resolve URLs, return the immediate result.
210 Pass in 'in_playlist' to only show this behavior for
211 playlist items.
212 postprocessors: A list of dictionaries, each with an entry
213 * key: The name of the postprocessor. See
214 youtube_dl/postprocessor/__init__.py for a list.
215 as well as any further keyword arguments for the
216 postprocessor.
217 progress_hooks: A list of functions that get called on download
218 progress, with a dictionary with the entries
219 * status: One of "downloading", "error", or "finished".
220 Check this first and ignore unknown values.
221
222 If status is one of "downloading", or "finished", the
223 following properties may also be present:
224 * filename: The final filename (always present)
225 * tmpfilename: The filename we're currently writing to
226 * downloaded_bytes: Bytes on disk
227 * total_bytes: Size of the whole file, None if unknown
228 * total_bytes_estimate: Guess of the eventual file size,
229 None if unavailable.
230 * elapsed: The number of seconds since download started.
231 * eta: The estimated time in seconds, None if unknown
232 * speed: The download speed in bytes/second, None if
233 unknown
234 * fragment_index: The counter of the currently
235 downloaded video fragment.
236 * fragment_count: The number of fragments (= individual
237 files that will be merged)
238
239 Progress hooks are guaranteed to be called at least once
240 (with status "finished") if the download is successful.
241 merge_output_format: Extension to use when merging formats.
242 fixup: Automatically correct known faults of the file.
243 One of:
244 - "never": do nothing
245 - "warn": only emit a warning
246 - "detect_or_warn": check whether we can do anything
247 about it, warn otherwise (default)
248 source_address: (Experimental) Client-side IP address to bind to.
249 call_home: Boolean, true iff we are allowed to contact the
250 youtube-dl servers for debugging.
251 sleep_interval: Number of seconds to sleep before each download.
252 listformats: Print an overview of available video formats and exit.
253 list_thumbnails: Print a table of all thumbnails and exit.
254 match_filter: A function that gets called with the info_dict of
255 every video.
256 If it returns a message, the video is ignored.
257 If it returns None, the video is downloaded.
258 match_filter_func in utils.py is one example for this.
259 no_color: Do not emit color codes in output.
260
261 The following options determine which downloader is picked:
262 external_downloader: Executable of the external downloader to call.
263 None or unset for standard (built-in) downloader.
264 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
265 if True, otherwise use ffmpeg/avconv if False, otherwise
266 use downloader suggested by extractor if None.
267
268 The following parameters are not used by YoutubeDL itself, they are used by
269 the downloader (see youtube_dl/downloader/common.py):
270 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
271 noresizebuffer, retries, continuedl, noprogress, consoletitle,
272 xattr_set_filesize, external_downloader_args, hls_use_mpegts.
273
274 The following options are used by the post processors:
275 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
276 otherwise prefer avconv.
277 postprocessor_args: A list of additional command-line arguments for the
278 postprocessor.
279 """
280
281 params = None
282 _ies = []
283 _pps = []
284 _download_retcode = None
285 _num_downloads = None
286 _screen_file = None
287
288 def __init__(self, params=None, auto_init=True):
289 """Create a FileDownloader object with the given options."""
290 if params is None:
291 params = {}
292 self._ies = []
293 self._ies_instances = {}
294 self._pps = []
295 self._progress_hooks = []
296 self._download_retcode = 0
297 self._num_downloads = 0
298 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
299 self._err_file = sys.stderr
300 self.params = {
301 # Default parameters
302 'nocheckcertificate': False,
303 }
304 self.params.update(params)
305 self.cache = Cache(self)
306
307 if params.get('bidi_workaround', False):
308 try:
309 import pty
310 master, slave = pty.openpty()
311 width = compat_get_terminal_size().columns
312 if width is None:
313 width_args = []
314 else:
315 width_args = ['-w', str(width)]
316 sp_kwargs = dict(
317 stdin=subprocess.PIPE,
318 stdout=slave,
319 stderr=self._err_file)
320 try:
321 self._output_process = subprocess.Popen(
322 ['bidiv'] + width_args, **sp_kwargs
323 )
324 except OSError:
325 self._output_process = subprocess.Popen(
326 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
327 self._output_channel = os.fdopen(master, 'rb')
328 except OSError as ose:
329 if ose.errno == errno.ENOENT:
330 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
331 else:
332 raise
333
334 if (sys.version_info >= (3,) and sys.platform != 'win32' and
335 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
336 not params.get('restrictfilenames', False)):
337 # On Python 3, the Unicode filesystem API will throw errors (#1474)
338 self.report_warning(
339 'Assuming --restrict-filenames since file system encoding '
340 'cannot encode all characters. '
341 'Set the LC_ALL environment variable to fix this.')
342 self.params['restrictfilenames'] = True
343
344 if isinstance(params.get('outtmpl'), bytes):
345 self.report_warning(
346 'Parameter outtmpl is bytes, but should be a unicode string. '
347 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
348
349 self._setup_opener()
350
351 if auto_init:
352 self.print_debug_header()
353 self.add_default_info_extractors()
354
355 for pp_def_raw in self.params.get('postprocessors', []):
356 pp_class = get_postprocessor(pp_def_raw['key'])
357 pp_def = dict(pp_def_raw)
358 del pp_def['key']
359 pp = pp_class(self, **compat_kwargs(pp_def))
360 self.add_post_processor(pp)
361
362 for ph in self.params.get('progress_hooks', []):
363 self.add_progress_hook(ph)
364
365 register_socks_protocols()
366
367 def warn_if_short_id(self, argv):
368 # short YouTube ID starting with dash?
369 idxs = [
370 i for i, a in enumerate(argv)
371 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
372 if idxs:
373 correct_argv = (
374 ['youtube-dl'] +
375 [a for i, a in enumerate(argv) if i not in idxs] +
376 ['--'] + [argv[i] for i in idxs]
377 )
378 self.report_warning(
379 'Long argument string detected. '
380 'Use -- to separate parameters and URLs, like this:\n%s\n' %
381 args_to_str(correct_argv))
382
383 def add_info_extractor(self, ie):
384 """Add an InfoExtractor object to the end of the list."""
385 self._ies.append(ie)
386 if not isinstance(ie, type):
387 self._ies_instances[ie.ie_key()] = ie
388 ie.set_downloader(self)
389
390 def get_info_extractor(self, ie_key):
391 """
392 Get an instance of an IE with name ie_key, it will try to get one from
393 the _ies list, if there's no instance it will create a new one and add
394 it to the extractor list.
395 """
396 ie = self._ies_instances.get(ie_key)
397 if ie is None:
398 ie = get_info_extractor(ie_key)()
399 self.add_info_extractor(ie)
400 return ie
401
402 def add_default_info_extractors(self):
403 """
404 Add the InfoExtractors returned by gen_extractors to the end of the list
405 """
406 for ie in gen_extractor_classes():
407 self.add_info_extractor(ie)
408
409 def add_post_processor(self, pp):
410 """Add a PostProcessor object to the end of the chain."""
411 self._pps.append(pp)
412 pp.set_downloader(self)
413
414 def add_progress_hook(self, ph):
415 """Add the progress hook (currently only for the file downloader)"""
416 self._progress_hooks.append(ph)
417
418 def _bidi_workaround(self, message):
419 if not hasattr(self, '_output_channel'):
420 return message
421
422 assert hasattr(self, '_output_process')
423 assert isinstance(message, compat_str)
424 line_count = message.count('\n') + 1
425 self._output_process.stdin.write((message + '\n').encode('utf-8'))
426 self._output_process.stdin.flush()
427 res = ''.join(self._output_channel.readline().decode('utf-8')
428 for _ in range(line_count))
429 return res[:-len('\n')]
430
431 def to_screen(self, message, skip_eol=False):
432 """Print message to stdout if not in quiet mode."""
433 return self.to_stdout(message, skip_eol, check_quiet=True)
434
435 def _write_string(self, s, out=None):
436 write_string(s, out=out, encoding=self.params.get('encoding'))
437
438 def to_stdout(self, message, skip_eol=False, check_quiet=False):
439 """Print message to stdout if not in quiet mode."""
440 if self.params.get('logger'):
441 self.params['logger'].debug(message)
442 elif not check_quiet or not self.params.get('quiet', False):
443 message = self._bidi_workaround(message)
444 terminator = ['\n', ''][skip_eol]
445 output = message + terminator
446
447 self._write_string(output, self._screen_file)
448
449 def to_stderr(self, message):
450 """Print message to stderr."""
451 assert isinstance(message, compat_str)
452 if self.params.get('logger'):
453 self.params['logger'].error(message)
454 else:
455 message = self._bidi_workaround(message)
456 output = message + '\n'
457 self._write_string(output, self._err_file)
458
459 def to_console_title(self, message):
460 if not self.params.get('consoletitle', False):
461 return
462 if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
463 # c_wchar_p() might not be necessary if `message` is
464 # already of type unicode()
465 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
466 elif 'TERM' in os.environ:
467 self._write_string('\033]0;%s\007' % message, self._screen_file)
468
469 def save_console_title(self):
470 if not self.params.get('consoletitle', False):
471 return
472 if 'TERM' in os.environ:
473 # Save the title on stack
474 self._write_string('\033[22;0t', self._screen_file)
475
476 def restore_console_title(self):
477 if not self.params.get('consoletitle', False):
478 return
479 if 'TERM' in os.environ:
480 # Restore the title from stack
481 self._write_string('\033[23;0t', self._screen_file)
482
483 def __enter__(self):
484 self.save_console_title()
485 return self
486
487 def __exit__(self, *args):
488 self.restore_console_title()
489
490 if self.params.get('cookiefile') is not None:
491 self.cookiejar.save()
492
493 def trouble(self, message=None, tb=None):
494 """Determine action to take when a download problem appears.
495
496 Depending on if the downloader has been configured to ignore
497 download errors or not, this method may throw an exception or
498 not when errors are found, after printing the message.
499
500 tb, if given, is additional traceback information.
501 """
502 if message is not None:
503 self.to_stderr(message)
504 if self.params.get('verbose'):
505 if tb is None:
506 if sys.exc_info()[0]: # if .trouble has been called from an except block
507 tb = ''
508 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
509 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
510 tb += encode_compat_str(traceback.format_exc())
511 else:
512 tb_data = traceback.format_list(traceback.extract_stack())
513 tb = ''.join(tb_data)
514 self.to_stderr(tb)
515 if not self.params.get('ignoreerrors', False):
516 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
517 exc_info = sys.exc_info()[1].exc_info
518 else:
519 exc_info = sys.exc_info()
520 raise DownloadError(message, exc_info)
521 self._download_retcode = 1
522
523 def report_warning(self, message):
524 '''
525 Print the message to stderr, it will be prefixed with 'WARNING:'
526 If stderr is a tty file the 'WARNING:' will be colored
527 '''
528 if self.params.get('logger') is not None:
529 self.params['logger'].warning(message)
530 else:
531 if self.params.get('no_warnings'):
532 return
533 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
534 _msg_header = '\033[0;33mWARNING:\033[0m'
535 else:
536 _msg_header = 'WARNING:'
537 warning_message = '%s %s' % (_msg_header, message)
538 self.to_stderr(warning_message)
539
540 def report_error(self, message, tb=None):
541 '''
542 Do the same as trouble, but prefixes the message with 'ERROR:', colored
543 in red if stderr is a tty file.
544 '''
545 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
546 _msg_header = '\033[0;31mERROR:\033[0m'
547 else:
548 _msg_header = 'ERROR:'
549 error_message = '%s %s' % (_msg_header, message)
550 self.trouble(error_message, tb)
551
552 def report_file_already_downloaded(self, file_name):
553 """Report file has already been fully downloaded."""
554 try:
555 self.to_screen('[download] %s has already been downloaded' % file_name)
556 except UnicodeEncodeError:
557 self.to_screen('[download] The file has already been downloaded')
558
559 def prepare_filename(self, info_dict):
560 """Generate the output filename."""
561 try:
562 template_dict = dict(info_dict)
563
564 template_dict['epoch'] = int(time.time())
565 autonumber_size = self.params.get('autonumber_size')
566 if autonumber_size is None:
567 autonumber_size = 5
568 autonumber_templ = '%0' + str(autonumber_size) + 'd'
569 template_dict['autonumber'] = autonumber_templ % self._num_downloads
570 if template_dict.get('playlist_index') is not None:
571 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
572 if template_dict.get('resolution') is None:
573 if template_dict.get('width') and template_dict.get('height'):
574 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
575 elif template_dict.get('height'):
576 template_dict['resolution'] = '%sp' % template_dict['height']
577 elif template_dict.get('width'):
578 template_dict['resolution'] = '%dx?' % template_dict['width']
579
580 sanitize = lambda k, v: sanitize_filename(
581 compat_str(v),
582 restricted=self.params.get('restrictfilenames'),
583 is_id=(k == 'id'))
584 template_dict = dict((k, sanitize(k, v))
585 for k, v in template_dict.items()
586 if v is not None and not isinstance(v, (list, tuple, dict)))
587 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
588
589 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
590 tmpl = compat_expanduser(outtmpl)
591 filename = tmpl % template_dict
592 # Temporary fix for #4787
593 # 'Treat' all problem characters by passing filename through preferredencoding
594 # to workaround encoding issues with subprocess on python2 @ Windows
595 if sys.version_info < (3, 0) and sys.platform == 'win32':
596 filename = encodeFilename(filename, True).decode(preferredencoding())
597 return sanitize_path(filename)
598 except ValueError as err:
599 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
600 return None
601
602 def _match_entry(self, info_dict, incomplete):
603 """ Returns None iff the file should be downloaded """
604
605 video_title = info_dict.get('title', info_dict.get('id', 'video'))
606 if 'title' in info_dict:
607 # This can happen when we're just evaluating the playlist
608 title = info_dict['title']
609 matchtitle = self.params.get('matchtitle', False)
610 if matchtitle:
611 if not re.search(matchtitle, title, re.IGNORECASE):
612 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
613 rejecttitle = self.params.get('rejecttitle', False)
614 if rejecttitle:
615 if re.search(rejecttitle, title, re.IGNORECASE):
616 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
617 date = info_dict.get('upload_date')
618 if date is not None:
619 dateRange = self.params.get('daterange', DateRange())
620 if date not in dateRange:
621 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
622 view_count = info_dict.get('view_count')
623 if view_count is not None:
624 min_views = self.params.get('min_views')
625 if min_views is not None and view_count < min_views:
626 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
627 max_views = self.params.get('max_views')
628 if max_views is not None and view_count > max_views:
629 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
630 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
631 return 'Skipping "%s" because it is age restricted' % video_title
632 if self.in_download_archive(info_dict):
633 return '%s has already been recorded in archive' % video_title
634
635 if not incomplete:
636 match_filter = self.params.get('match_filter')
637 if match_filter is not None:
638 ret = match_filter(info_dict)
639 if ret is not None:
640 return ret
641
642 return None
643
644 @staticmethod
645 def add_extra_info(info_dict, extra_info):
646 '''Set the keys from extra_info in info dict if they are missing'''
647 for key, value in extra_info.items():
648 info_dict.setdefault(key, value)
649
650 def extract_info(self, url, download=True, ie_key=None, extra_info={},
651 process=True, force_generic_extractor=False):
652 '''
653 Returns a list with a dictionary for each video we find.
654 If 'download', also downloads the videos.
655 extra_info is a dict containing the extra values to add to each result
656 '''
657
658 if not ie_key and force_generic_extractor:
659 ie_key = 'Generic'
660
661 if ie_key:
662 ies = [self.get_info_extractor(ie_key)]
663 else:
664 ies = self._ies
665
666 for ie in ies:
667 if not ie.suitable(url):
668 continue
669
670 ie = self.get_info_extractor(ie.ie_key())
671 if not ie.working():
672 self.report_warning('The program functionality for this site has been marked as broken, '
673 'and will probably not work.')
674
675 try:
676 ie_result = ie.extract(url)
677 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
678 break
679 if isinstance(ie_result, list):
680 # Backwards compatibility: old IE result format
681 ie_result = {
682 '_type': 'compat_list',
683 'entries': ie_result,
684 }
685 self.add_default_extra_info(ie_result, ie, url)
686 if process:
687 return self.process_ie_result(ie_result, download, extra_info)
688 else:
689 return ie_result
690 except ExtractorError as e: # An error we somewhat expected
691 self.report_error(compat_str(e), e.format_traceback())
692 break
693 except MaxDownloadsReached:
694 raise
695 except Exception as e:
696 if self.params.get('ignoreerrors', False):
697 self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
698 break
699 else:
700 raise
701 else:
702 self.report_error('no suitable InfoExtractor for URL %s' % url)
703
704 def add_default_extra_info(self, ie_result, ie, url):
705 self.add_extra_info(ie_result, {
706 'extractor': ie.IE_NAME,
707 'webpage_url': url,
708 'webpage_url_basename': url_basename(url),
709 'extractor_key': ie.ie_key(),
710 })
711
712 def process_ie_result(self, ie_result, download=True, extra_info={}):
713 """
714 Take the result of the ie(may be modified) and resolve all unresolved
715 references (URLs, playlist items).
716
717 It will also download the videos if 'download'.
718 Returns the resolved ie_result.
719 """
720 result_type = ie_result.get('_type', 'video')
721
722 if result_type in ('url', 'url_transparent'):
723 ie_result['url'] = sanitize_url(ie_result['url'])
724 extract_flat = self.params.get('extract_flat', False)
725 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
726 extract_flat is True):
727 if self.params.get('forcejson', False):
728 self.to_stdout(json.dumps(ie_result))
729 return ie_result
730
731 if result_type == 'video':
732 self.add_extra_info(ie_result, extra_info)
733 return self.process_video_result(ie_result, download=download)
734 elif result_type == 'url':
735 # We have to add extra_info to the results because it may be
736 # contained in a playlist
737 return self.extract_info(ie_result['url'],
738 download,
739 ie_key=ie_result.get('ie_key'),
740 extra_info=extra_info)
741 elif result_type == 'url_transparent':
742 # Use the information from the embedding page
743 info = self.extract_info(
744 ie_result['url'], ie_key=ie_result.get('ie_key'),
745 extra_info=extra_info, download=False, process=False)
746
747 force_properties = dict(
748 (k, v) for k, v in ie_result.items() if v is not None)
749 for f in ('_type', 'url', 'ie_key'):
750 if f in force_properties:
751 del force_properties[f]
752 new_result = info.copy()
753 new_result.update(force_properties)
754
755 assert new_result.get('_type') != 'url_transparent'
756
757 return self.process_ie_result(
758 new_result, download=download, extra_info=extra_info)
759 elif result_type == 'playlist' or result_type == 'multi_video':
760 # We process each entry in the playlist
761 playlist = ie_result.get('title') or ie_result.get('id')
762 self.to_screen('[download] Downloading playlist: %s' % playlist)
763
764 playlist_results = []
765
766 playliststart = self.params.get('playliststart', 1) - 1
767 playlistend = self.params.get('playlistend')
768 # For backwards compatibility, interpret -1 as whole list
769 if playlistend == -1:
770 playlistend = None
771
772 playlistitems_str = self.params.get('playlist_items')
773 playlistitems = None
774 if playlistitems_str is not None:
775 def iter_playlistitems(format):
776 for string_segment in format.split(','):
777 if '-' in string_segment:
778 start, end = string_segment.split('-')
779 for item in range(int(start), int(end) + 1):
780 yield int(item)
781 else:
782 yield int(string_segment)
783 playlistitems = iter_playlistitems(playlistitems_str)
784
785 ie_entries = ie_result['entries']
786 if isinstance(ie_entries, list):
787 n_all_entries = len(ie_entries)
788 if playlistitems:
789 entries = [
790 ie_entries[i - 1] for i in playlistitems
791 if -n_all_entries <= i - 1 < n_all_entries]
792 else:
793 entries = ie_entries[playliststart:playlistend]
794 n_entries = len(entries)
795 self.to_screen(
796 '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
797 (ie_result['extractor'], playlist, n_all_entries, n_entries))
798 elif isinstance(ie_entries, PagedList):
799 if playlistitems:
800 entries = []
801 for item in playlistitems:
802 entries.extend(ie_entries.getslice(
803 item - 1, item
804 ))
805 else:
806 entries = ie_entries.getslice(
807 playliststart, playlistend)
808 n_entries = len(entries)
809 self.to_screen(
810 '[%s] playlist %s: Downloading %d videos' %
811 (ie_result['extractor'], playlist, n_entries))
812 else: # iterable
813 if playlistitems:
814 entry_list = list(ie_entries)
815 entries = [entry_list[i - 1] for i in playlistitems]
816 else:
817 entries = list(itertools.islice(
818 ie_entries, playliststart, playlistend))
819 n_entries = len(entries)
820 self.to_screen(
821 '[%s] playlist %s: Downloading %d videos' %
822 (ie_result['extractor'], playlist, n_entries))
823
824 if self.params.get('playlistreverse', False):
825 entries = entries[::-1]
826
827 for i, entry in enumerate(entries, 1):
828 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
829 extra = {
830 'n_entries': n_entries,
831 'playlist': playlist,
832 'playlist_id': ie_result.get('id'),
833 'playlist_title': ie_result.get('title'),
834 'playlist_index': i + playliststart,
835 'extractor': ie_result['extractor'],
836 'webpage_url': ie_result['webpage_url'],
837 'webpage_url_basename': url_basename(ie_result['webpage_url']),
838 'extractor_key': ie_result['extractor_key'],
839 }
840
841 reason = self._match_entry(entry, incomplete=True)
842 if reason is not None:
843 self.to_screen('[download] ' + reason)
844 continue
845
846 entry_result = self.process_ie_result(entry,
847 download=download,
848 extra_info=extra)
849 playlist_results.append(entry_result)
850 ie_result['entries'] = playlist_results
851 self.to_screen('[download] Finished downloading playlist: %s' % playlist)
852 return ie_result
853 elif result_type == 'compat_list':
854 self.report_warning(
855 'Extractor %s returned a compat_list result. '
856 'It needs to be updated.' % ie_result.get('extractor'))
857
858 def _fixup(r):
859 self.add_extra_info(
860 r,
861 {
862 'extractor': ie_result['extractor'],
863 'webpage_url': ie_result['webpage_url'],
864 'webpage_url_basename': url_basename(ie_result['webpage_url']),
865 'extractor_key': ie_result['extractor_key'],
866 }
867 )
868 return r
869 ie_result['entries'] = [
870 self.process_ie_result(_fixup(r), download, extra_info)
871 for r in ie_result['entries']
872 ]
873 return ie_result
874 else:
875 raise Exception('Invalid result type: %s' % result_type)
876
877 def _build_format_filter(self, filter_spec):
878 " Returns a function to filter the formats according to the filter_spec "
879
880 OPERATORS = {
881 '<': operator.lt,
882 '<=': operator.le,
883 '>': operator.gt,
884 '>=': operator.ge,
885 '=': operator.eq,
886 '!=': operator.ne,
887 }
888 operator_rex = re.compile(r'''(?x)\s*
889 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
890 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
891 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
892 $
893 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
894 m = operator_rex.search(filter_spec)
895 if m:
896 try:
897 comparison_value = int(m.group('value'))
898 except ValueError:
899 comparison_value = parse_filesize(m.group('value'))
900 if comparison_value is None:
901 comparison_value = parse_filesize(m.group('value') + 'B')
902 if comparison_value is None:
903 raise ValueError(
904 'Invalid value %r in format specification %r' % (
905 m.group('value'), filter_spec))
906 op = OPERATORS[m.group('op')]
907
908 if not m:
909 STR_OPERATORS = {
910 '=': operator.eq,
911 '!=': operator.ne,
912 '^=': lambda attr, value: attr.startswith(value),
913 '$=': lambda attr, value: attr.endswith(value),
914 '*=': lambda attr, value: value in attr,
915 }
916 str_operator_rex = re.compile(r'''(?x)
917 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
918 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
919 \s*(?P<value>[a-zA-Z0-9._-]+)
920 \s*$
921 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
922 m = str_operator_rex.search(filter_spec)
923 if m:
924 comparison_value = m.group('value')
925 op = STR_OPERATORS[m.group('op')]
926
927 if not m:
928 raise ValueError('Invalid filter specification %r' % filter_spec)
929
930 def _filter(f):
931 actual_value = f.get(m.group('key'))
932 if actual_value is None:
933 return m.group('none_inclusive')
934 return op(actual_value, comparison_value)
935 return _filter
936
937 def build_format_selector(self, format_spec):
938 def syntax_error(note, start):
939 message = (
940 'Invalid format specification: '
941 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
942 return SyntaxError(message)
943
944 PICKFIRST = 'PICKFIRST'
945 MERGE = 'MERGE'
946 SINGLE = 'SINGLE'
947 GROUP = 'GROUP'
948 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
949
950 def _parse_filter(tokens):
951 filter_parts = []
952 for type, string, start, _, _ in tokens:
953 if type == tokenize.OP and string == ']':
954 return ''.join(filter_parts)
955 else:
956 filter_parts.append(string)
957
958 def _remove_unused_ops(tokens):
959 # Remove operators that we don't use and join them with the surrounding strings
960 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
961 ALLOWED_OPS = ('/', '+', ',', '(', ')')
962 last_string, last_start, last_end, last_line = None, None, None, None
963 for type, string, start, end, line in tokens:
964 if type == tokenize.OP and string == '[':
965 if last_string:
966 yield tokenize.NAME, last_string, last_start, last_end, last_line
967 last_string = None
968 yield type, string, start, end, line
969 # everything inside brackets will be handled by _parse_filter
970 for type, string, start, end, line in tokens:
971 yield type, string, start, end, line
972 if type == tokenize.OP and string == ']':
973 break
974 elif type == tokenize.OP and string in ALLOWED_OPS:
975 if last_string:
976 yield tokenize.NAME, last_string, last_start, last_end, last_line
977 last_string = None
978 yield type, string, start, end, line
979 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
980 if not last_string:
981 last_string = string
982 last_start = start
983 last_end = end
984 else:
985 last_string += string
986 if last_string:
987 yield tokenize.NAME, last_string, last_start, last_end, last_line
988
989 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
990 selectors = []
991 current_selector = None
992 for type, string, start, _, _ in tokens:
993 # ENCODING is only defined in python 3.x
994 if type == getattr(tokenize, 'ENCODING', None):
995 continue
996 elif type in [tokenize.NAME, tokenize.NUMBER]:
997 current_selector = FormatSelector(SINGLE, string, [])
998 elif type == tokenize.OP:
999 if string == ')':
1000 if not inside_group:
1001 # ')' will be handled by the parentheses group
1002 tokens.restore_last_token()
1003 break
1004 elif inside_merge and string in ['/', ',']:
1005 tokens.restore_last_token()
1006 break
1007 elif inside_choice and string == ',':
1008 tokens.restore_last_token()
1009 break
1010 elif string == ',':
1011 if not current_selector:
1012 raise syntax_error('"," must follow a format selector', start)
1013 selectors.append(current_selector)
1014 current_selector = None
1015 elif string == '/':
1016 if not current_selector:
1017 raise syntax_error('"/" must follow a format selector', start)
1018 first_choice = current_selector
1019 second_choice = _parse_format_selection(tokens, inside_choice=True)
1020 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1021 elif string == '[':
1022 if not current_selector:
1023 current_selector = FormatSelector(SINGLE, 'best', [])
1024 format_filter = _parse_filter(tokens)
1025 current_selector.filters.append(format_filter)
1026 elif string == '(':
1027 if current_selector:
1028 raise syntax_error('Unexpected "("', start)
1029 group = _parse_format_selection(tokens, inside_group=True)
1030 current_selector = FormatSelector(GROUP, group, [])
1031 elif string == '+':
1032 video_selector = current_selector
1033 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1034 if not video_selector or not audio_selector:
1035 raise syntax_error('"+" must be between two format selectors', start)
1036 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1037 else:
1038 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1039 elif type == tokenize.ENDMARKER:
1040 break
1041 if current_selector:
1042 selectors.append(current_selector)
1043 return selectors
1044
1045 def _build_selector_function(selector):
1046 if isinstance(selector, list):
1047 fs = [_build_selector_function(s) for s in selector]
1048
1049 def selector_function(formats):
1050 for f in fs:
1051 for format in f(formats):
1052 yield format
1053 return selector_function
1054 elif selector.type == GROUP:
1055 selector_function = _build_selector_function(selector.selector)
1056 elif selector.type == PICKFIRST:
1057 fs = [_build_selector_function(s) for s in selector.selector]
1058
1059 def selector_function(formats):
1060 for f in fs:
1061 picked_formats = list(f(formats))
1062 if picked_formats:
1063 return picked_formats
1064 return []
1065 elif selector.type == SINGLE:
1066 format_spec = selector.selector
1067
1068 def selector_function(formats):
1069 formats = list(formats)
1070 if not formats:
1071 return
1072 if format_spec == 'all':
1073 for f in formats:
1074 yield f
1075 elif format_spec in ['best', 'worst', None]:
1076 format_idx = 0 if format_spec == 'worst' else -1
1077 audiovideo_formats = [
1078 f for f in formats
1079 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1080 if audiovideo_formats:
1081 yield audiovideo_formats[format_idx]
1082 # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1083 elif (all(f.get('acodec') != 'none' for f in formats) or
1084 all(f.get('vcodec') != 'none' for f in formats)):
1085 yield formats[format_idx]
1086 elif format_spec == 'bestaudio':
1087 audio_formats = [
1088 f for f in formats
1089 if f.get('vcodec') == 'none']
1090 if audio_formats:
1091 yield audio_formats[-1]
1092 elif format_spec == 'worstaudio':
1093 audio_formats = [
1094 f for f in formats
1095 if f.get('vcodec') == 'none']
1096 if audio_formats:
1097 yield audio_formats[0]
1098 elif format_spec == 'bestvideo':
1099 video_formats = [
1100 f for f in formats
1101 if f.get('acodec') == 'none']
1102 if video_formats:
1103 yield video_formats[-1]
1104 elif format_spec == 'worstvideo':
1105 video_formats = [
1106 f for f in formats
1107 if f.get('acodec') == 'none']
1108 if video_formats:
1109 yield video_formats[0]
1110 else:
1111 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1112 if format_spec in extensions:
1113 filter_f = lambda f: f['ext'] == format_spec
1114 else:
1115 filter_f = lambda f: f['format_id'] == format_spec
1116 matches = list(filter(filter_f, formats))
1117 if matches:
1118 yield matches[-1]
1119 elif selector.type == MERGE:
1120 def _merge(formats_info):
1121 format_1, format_2 = [f['format_id'] for f in formats_info]
1122 # The first format must contain the video and the
1123 # second the audio
1124 if formats_info[0].get('vcodec') == 'none':
1125 self.report_error('The first format must '
1126 'contain the video, try using '
1127 '"-f %s+%s"' % (format_2, format_1))
1128 return
1129 # Formats must be opposite (video+audio)
1130 if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1131 self.report_error(
1132 'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1133 % (format_1, format_2))
1134 return
1135 output_ext = (
1136 formats_info[0]['ext']
1137 if self.params.get('merge_output_format') is None
1138 else self.params['merge_output_format'])
1139 return {
1140 'requested_formats': formats_info,
1141 'format': '%s+%s' % (formats_info[0].get('format'),
1142 formats_info[1].get('format')),
1143 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1144 formats_info[1].get('format_id')),
1145 'width': formats_info[0].get('width'),
1146 'height': formats_info[0].get('height'),
1147 'resolution': formats_info[0].get('resolution'),
1148 'fps': formats_info[0].get('fps'),
1149 'vcodec': formats_info[0].get('vcodec'),
1150 'vbr': formats_info[0].get('vbr'),
1151 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1152 'acodec': formats_info[1].get('acodec'),
1153 'abr': formats_info[1].get('abr'),
1154 'ext': output_ext,
1155 }
1156 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1157
1158 def selector_function(formats):
1159 formats = list(formats)
1160 for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1161 yield _merge(pair)
1162
1163 filters = [self._build_format_filter(f) for f in selector.filters]
1164
1165 def final_selector(formats):
1166 for _filter in filters:
1167 formats = list(filter(_filter, formats))
1168 return selector_function(formats)
1169 return final_selector
1170
1171 stream = io.BytesIO(format_spec.encode('utf-8'))
1172 try:
1173 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1174 except tokenize.TokenError:
1175 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1176
1177 class TokenIterator(object):
1178 def __init__(self, tokens):
1179 self.tokens = tokens
1180 self.counter = 0
1181
1182 def __iter__(self):
1183 return self
1184
1185 def __next__(self):
1186 if self.counter >= len(self.tokens):
1187 raise StopIteration()
1188 value = self.tokens[self.counter]
1189 self.counter += 1
1190 return value
1191
1192 next = __next__
1193
1194 def restore_last_token(self):
1195 self.counter -= 1
1196
1197 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1198 return _build_selector_function(parsed_selector)
1199
1200 def _calc_headers(self, info_dict):
1201 res = std_headers.copy()
1202
1203 add_headers = info_dict.get('http_headers')
1204 if add_headers:
1205 res.update(add_headers)
1206
1207 cookies = self._calc_cookies(info_dict)
1208 if cookies:
1209 res['Cookie'] = cookies
1210
1211 return res
1212
1213 def _calc_cookies(self, info_dict):
1214 pr = sanitized_Request(info_dict['url'])
1215 self.cookiejar.add_cookie_header(pr)
1216 return pr.get_header('Cookie')
1217
1218 def process_video_result(self, info_dict, download=True):
1219 assert info_dict.get('_type', 'video') == 'video'
1220
1221 if 'id' not in info_dict:
1222 raise ExtractorError('Missing "id" field in extractor result')
1223 if 'title' not in info_dict:
1224 raise ExtractorError('Missing "title" field in extractor result')
1225
1226 if not isinstance(info_dict['id'], compat_str):
1227 self.report_warning('"id" field is not a string - forcing string conversion')
1228 info_dict['id'] = compat_str(info_dict['id'])
1229
1230 if 'playlist' not in info_dict:
1231 # It isn't part of a playlist
1232 info_dict['playlist'] = None
1233 info_dict['playlist_index'] = None
1234
1235 thumbnails = info_dict.get('thumbnails')
1236 if thumbnails is None:
1237 thumbnail = info_dict.get('thumbnail')
1238 if thumbnail:
1239 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1240 if thumbnails:
1241 thumbnails.sort(key=lambda t: (
1242 t.get('preference'), t.get('width'), t.get('height'),
1243 t.get('id'), t.get('url')))
1244 for i, t in enumerate(thumbnails):
1245 t['url'] = sanitize_url(t['url'])
1246 if t.get('width') and t.get('height'):
1247 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1248 if t.get('id') is None:
1249 t['id'] = '%d' % i
1250
1251 if self.params.get('list_thumbnails'):
1252 self.list_thumbnails(info_dict)
1253 return
1254
1255 thumbnail = info_dict.get('thumbnail')
1256 if thumbnail:
1257 info_dict['thumbnail'] = sanitize_url(thumbnail)
1258 elif thumbnails:
1259 info_dict['thumbnail'] = thumbnails[-1]['url']
1260
1261 if 'display_id' not in info_dict and 'id' in info_dict:
1262 info_dict['display_id'] = info_dict['id']
1263
1264 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1265 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1266 # see http://bugs.python.org/issue1646728)
1267 try:
1268 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1269 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1270 except (ValueError, OverflowError, OSError):
1271 pass
1272
1273 # Auto generate title fields corresponding to the *_number fields when missing
1274 # in order to always have clean titles. This is very common for TV series.
1275 for field in ('chapter', 'season', 'episode'):
1276 if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1277 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1278
1279 subtitles = info_dict.get('subtitles')
1280 if subtitles:
1281 for _, subtitle in subtitles.items():
1282 for subtitle_format in subtitle:
1283 if subtitle_format.get('url'):
1284 subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1285 if 'ext' not in subtitle_format:
1286 subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1287
1288 if self.params.get('listsubtitles', False):
1289 if 'automatic_captions' in info_dict:
1290 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1291 self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1292 return
1293 info_dict['requested_subtitles'] = self.process_subtitles(
1294 info_dict['id'], subtitles,
1295 info_dict.get('automatic_captions'))
1296
1297 # We now pick which formats have to be downloaded
1298 if info_dict.get('formats') is None:
1299 # There's only one format available
1300 formats = [info_dict]
1301 else:
1302 formats = info_dict['formats']
1303
1304 if not formats:
1305 raise ExtractorError('No video formats found!')
1306
1307 formats_dict = {}
1308
1309 # We check that all the formats have the format and format_id fields
1310 for i, format in enumerate(formats):
1311 if 'url' not in format:
1312 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1313
1314 format['url'] = sanitize_url(format['url'])
1315
1316 if format.get('format_id') is None:
1317 format['format_id'] = compat_str(i)
1318 else:
1319 # Sanitize format_id from characters used in format selector expression
1320 format['format_id'] = re.sub('[\s,/+\[\]()]', '_', format['format_id'])
1321 format_id = format['format_id']
1322 if format_id not in formats_dict:
1323 formats_dict[format_id] = []
1324 formats_dict[format_id].append(format)
1325
1326 # Make sure all formats have unique format_id
1327 for format_id, ambiguous_formats in formats_dict.items():
1328 if len(ambiguous_formats) > 1:
1329 for i, format in enumerate(ambiguous_formats):
1330 format['format_id'] = '%s-%d' % (format_id, i)
1331
1332 for i, format in enumerate(formats):
1333 if format.get('format') is None:
1334 format['format'] = '{id} - {res}{note}'.format(
1335 id=format['format_id'],
1336 res=self.format_resolution(format),
1337 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1338 )
1339 # Automatically determine file extension if missing
1340 if 'ext' not in format:
1341 format['ext'] = determine_ext(format['url']).lower()
1342 # Automatically determine protocol if missing (useful for format
1343 # selection purposes)
1344 if 'protocol' not in format:
1345 format['protocol'] = determine_protocol(format)
1346 # Add HTTP headers, so that external programs can use them from the
1347 # json output
1348 full_format_info = info_dict.copy()
1349 full_format_info.update(format)
1350 format['http_headers'] = self._calc_headers(full_format_info)
1351
1352 # TODO Central sorting goes here
1353
1354 if formats[0] is not info_dict:
1355 # only set the 'formats' fields if the original info_dict list them
1356 # otherwise we end up with a circular reference, the first (and unique)
1357 # element in the 'formats' field in info_dict is info_dict itself,
1358 # which can't be exported to json
1359 info_dict['formats'] = formats
1360 if self.params.get('listformats'):
1361 self.list_formats(info_dict)
1362 return
1363
1364 req_format = self.params.get('format')
1365 if req_format is None:
1366 req_format_list = []
1367 if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1368 not info_dict.get('is_live')):
1369 merger = FFmpegMergerPP(self)
1370 if merger.available and merger.can_merge():
1371 req_format_list.append('bestvideo+bestaudio')
1372 req_format_list.append('best')
1373 req_format = '/'.join(req_format_list)
1374 format_selector = self.build_format_selector(req_format)
1375 formats_to_download = list(format_selector(formats))
1376 if not formats_to_download:
1377 raise ExtractorError('requested format not available',
1378 expected=True)
1379
1380 if download:
1381 if len(formats_to_download) > 1:
1382 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1383 for format in formats_to_download:
1384 new_info = dict(info_dict)
1385 new_info.update(format)
1386 self.process_info(new_info)
1387 # We update the info dict with the best quality format (backwards compatibility)
1388 info_dict.update(formats_to_download[-1])
1389 return info_dict
1390
1391 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1392 """Select the requested subtitles and their format"""
1393 available_subs = {}
1394 if normal_subtitles and self.params.get('writesubtitles'):
1395 available_subs.update(normal_subtitles)
1396 if automatic_captions and self.params.get('writeautomaticsub'):
1397 for lang, cap_info in automatic_captions.items():
1398 if lang not in available_subs:
1399 available_subs[lang] = cap_info
1400
1401 if (not self.params.get('writesubtitles') and not
1402 self.params.get('writeautomaticsub') or not
1403 available_subs):
1404 return None
1405
1406 if self.params.get('allsubtitles', False):
1407 requested_langs = available_subs.keys()
1408 else:
1409 if self.params.get('subtitleslangs', False):
1410 requested_langs = self.params.get('subtitleslangs')
1411 elif 'en' in available_subs:
1412 requested_langs = ['en']
1413 else:
1414 requested_langs = [list(available_subs.keys())[0]]
1415
1416 formats_query = self.params.get('subtitlesformat', 'best')
1417 formats_preference = formats_query.split('/') if formats_query else []
1418 subs = {}
1419 for lang in requested_langs:
1420 formats = available_subs.get(lang)
1421 if formats is None:
1422 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1423 continue
1424 for ext in formats_preference:
1425 if ext == 'best':
1426 f = formats[-1]
1427 break
1428 matches = list(filter(lambda f: f['ext'] == ext, formats))
1429 if matches:
1430 f = matches[-1]
1431 break
1432 else:
1433 f = formats[-1]
1434 self.report_warning(
1435 'No subtitle format found matching "%s" for language %s, '
1436 'using %s' % (formats_query, lang, f['ext']))
1437 subs[lang] = f
1438 return subs
1439
1440 def process_info(self, info_dict):
1441 """Process a single resolved IE result."""
1442
1443 assert info_dict.get('_type', 'video') == 'video'
1444
1445 max_downloads = self.params.get('max_downloads')
1446 if max_downloads is not None:
1447 if self._num_downloads >= int(max_downloads):
1448 raise MaxDownloadsReached()
1449
1450 info_dict['fulltitle'] = info_dict['title']
1451 if len(info_dict['title']) > 200:
1452 info_dict['title'] = info_dict['title'][:197] + '...'
1453
1454 if 'format' not in info_dict:
1455 info_dict['format'] = info_dict['ext']
1456
1457 reason = self._match_entry(info_dict, incomplete=False)
1458 if reason is not None:
1459 self.to_screen('[download] ' + reason)
1460 return
1461
1462 self._num_downloads += 1
1463
1464 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1465
1466 # Forced printings
1467 if self.params.get('forcetitle', False):
1468 self.to_stdout(info_dict['fulltitle'])
1469 if self.params.get('forceid', False):
1470 self.to_stdout(info_dict['id'])
1471 if self.params.get('forceurl', False):
1472 if info_dict.get('requested_formats') is not None:
1473 for f in info_dict['requested_formats']:
1474 self.to_stdout(f['url'] + f.get('play_path', ''))
1475 else:
1476 # For RTMP URLs, also include the playpath
1477 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1478 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1479 self.to_stdout(info_dict['thumbnail'])
1480 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1481 self.to_stdout(info_dict['description'])
1482 if self.params.get('forcefilename', False) and filename is not None:
1483 self.to_stdout(filename)
1484 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1485 self.to_stdout(formatSeconds(info_dict['duration']))
1486 if self.params.get('forceformat', False):
1487 self.to_stdout(info_dict['format'])
1488 if self.params.get('forcejson', False):
1489 self.to_stdout(json.dumps(info_dict))
1490
1491 # Do nothing else if in simulate mode
1492 if self.params.get('simulate', False):
1493 return
1494
1495 if filename is None:
1496 return
1497
1498 try:
1499 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1500 if dn and not os.path.exists(dn):
1501 os.makedirs(dn)
1502 except (OSError, IOError) as err:
1503 self.report_error('unable to create directory ' + error_to_compat_str(err))
1504 return
1505
1506 if self.params.get('writedescription', False):
1507 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1508 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1509 self.to_screen('[info] Video description is already present')
1510 elif info_dict.get('description') is None:
1511 self.report_warning('There\'s no description to write.')
1512 else:
1513 try:
1514 self.to_screen('[info] Writing video description to: ' + descfn)
1515 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1516 descfile.write(info_dict['description'])
1517 except (OSError, IOError):
1518 self.report_error('Cannot write description file ' + descfn)
1519 return
1520
1521 if self.params.get('writeannotations', False):
1522 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1523 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1524 self.to_screen('[info] Video annotations are already present')
1525 else:
1526 try:
1527 self.to_screen('[info] Writing video annotations to: ' + annofn)
1528 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1529 annofile.write(info_dict['annotations'])
1530 except (KeyError, TypeError):
1531 self.report_warning('There are no annotations to write.')
1532 except (OSError, IOError):
1533 self.report_error('Cannot write annotations file: ' + annofn)
1534 return
1535
1536 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1537 self.params.get('writeautomaticsub')])
1538
1539 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1540 # subtitles download errors are already managed as troubles in relevant IE
1541 # that way it will silently go on when used with unsupporting IE
1542 subtitles = info_dict['requested_subtitles']
1543 ie = self.get_info_extractor(info_dict['extractor_key'])
1544 for sub_lang, sub_info in subtitles.items():
1545 sub_format = sub_info['ext']
1546 if sub_info.get('data') is not None:
1547 sub_data = sub_info['data']
1548 else:
1549 try:
1550 sub_data = ie._download_webpage(
1551 sub_info['url'], info_dict['id'], note=False)
1552 except ExtractorError as err:
1553 self.report_warning('Unable to download subtitle for "%s": %s' %
1554 (sub_lang, error_to_compat_str(err.cause)))
1555 continue
1556 try:
1557 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1558 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1559 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1560 else:
1561 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1562 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1563 subfile.write(sub_data)
1564 except (OSError, IOError):
1565 self.report_error('Cannot write subtitles file ' + sub_filename)
1566 return
1567
1568 if self.params.get('writeinfojson', False):
1569 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1570 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1571 self.to_screen('[info] Video description metadata is already present')
1572 else:
1573 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1574 try:
1575 write_json_file(self.filter_requested_info(info_dict), infofn)
1576 except (OSError, IOError):
1577 self.report_error('Cannot write metadata to JSON file ' + infofn)
1578 return
1579
1580 self._write_thumbnails(info_dict, filename)
1581
1582 if not self.params.get('skip_download', False):
1583 try:
1584 def dl(name, info):
1585 fd = get_suitable_downloader(info, self.params)(self, self.params)
1586 for ph in self._progress_hooks:
1587 fd.add_progress_hook(ph)
1588 if self.params.get('verbose'):
1589 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1590 return fd.download(name, info)
1591
1592 if info_dict.get('requested_formats') is not None:
1593 downloaded = []
1594 success = True
1595 merger = FFmpegMergerPP(self)
1596 if not merger.available:
1597 postprocessors = []
1598 self.report_warning('You have requested multiple '
1599 'formats but ffmpeg or avconv are not installed.'
1600 ' The formats won\'t be merged.')
1601 else:
1602 postprocessors = [merger]
1603
1604 def compatible_formats(formats):
1605 video, audio = formats
1606 # Check extension
1607 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1608 if video_ext and audio_ext:
1609 COMPATIBLE_EXTS = (
1610 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1611 ('webm')
1612 )
1613 for exts in COMPATIBLE_EXTS:
1614 if video_ext in exts and audio_ext in exts:
1615 return True
1616 # TODO: Check acodec/vcodec
1617 return False
1618
1619 filename_real_ext = os.path.splitext(filename)[1][1:]
1620 filename_wo_ext = (
1621 os.path.splitext(filename)[0]
1622 if filename_real_ext == info_dict['ext']
1623 else filename)
1624 requested_formats = info_dict['requested_formats']
1625 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1626 info_dict['ext'] = 'mkv'
1627 self.report_warning(
1628 'Requested formats are incompatible for merge and will be merged into mkv.')
1629 # Ensure filename always has a correct extension for successful merge
1630 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1631 if os.path.exists(encodeFilename(filename)):
1632 self.to_screen(
1633 '[download] %s has already been downloaded and '
1634 'merged' % filename)
1635 else:
1636 for f in requested_formats:
1637 new_info = dict(info_dict)
1638 new_info.update(f)
1639 fname = self.prepare_filename(new_info)
1640 fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1641 downloaded.append(fname)
1642 partial_success = dl(fname, new_info)
1643 success = success and partial_success
1644 info_dict['__postprocessors'] = postprocessors
1645 info_dict['__files_to_merge'] = downloaded
1646 else:
1647 # Just a single file
1648 success = dl(filename, info_dict)
1649 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1650 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1651 return
1652 except (OSError, IOError) as err:
1653 raise UnavailableVideoError(err)
1654 except (ContentTooShortError, ) as err:
1655 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1656 return
1657
1658 if success and filename != '-':
1659 # Fixup content
1660 fixup_policy = self.params.get('fixup')
1661 if fixup_policy is None:
1662 fixup_policy = 'detect_or_warn'
1663
1664 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1665
1666 stretched_ratio = info_dict.get('stretched_ratio')
1667 if stretched_ratio is not None and stretched_ratio != 1:
1668 if fixup_policy == 'warn':
1669 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1670 info_dict['id'], stretched_ratio))
1671 elif fixup_policy == 'detect_or_warn':
1672 stretched_pp = FFmpegFixupStretchedPP(self)
1673 if stretched_pp.available:
1674 info_dict.setdefault('__postprocessors', [])
1675 info_dict['__postprocessors'].append(stretched_pp)
1676 else:
1677 self.report_warning(
1678 '%s: Non-uniform pixel ratio (%s). %s'
1679 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1680 else:
1681 assert fixup_policy in ('ignore', 'never')
1682
1683 if (info_dict.get('requested_formats') is None and
1684 info_dict.get('container') == 'm4a_dash'):
1685 if fixup_policy == 'warn':
1686 self.report_warning(
1687 '%s: writing DASH m4a. '
1688 'Only some players support this container.'
1689 % info_dict['id'])
1690 elif fixup_policy == 'detect_or_warn':
1691 fixup_pp = FFmpegFixupM4aPP(self)
1692 if fixup_pp.available:
1693 info_dict.setdefault('__postprocessors', [])
1694 info_dict['__postprocessors'].append(fixup_pp)
1695 else:
1696 self.report_warning(
1697 '%s: writing DASH m4a. '
1698 'Only some players support this container. %s'
1699 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1700 else:
1701 assert fixup_policy in ('ignore', 'never')
1702
1703 if (info_dict.get('protocol') == 'm3u8_native' or
1704 info_dict.get('protocol') == 'm3u8' and
1705 self.params.get('hls_prefer_native')):
1706 if fixup_policy == 'warn':
1707 self.report_warning('%s: malformated aac bitstream.' % (
1708 info_dict['id']))
1709 elif fixup_policy == 'detect_or_warn':
1710 fixup_pp = FFmpegFixupM3u8PP(self)
1711 if fixup_pp.available:
1712 info_dict.setdefault('__postprocessors', [])
1713 info_dict['__postprocessors'].append(fixup_pp)
1714 else:
1715 self.report_warning(
1716 '%s: malformated aac bitstream. %s'
1717 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1718 else:
1719 assert fixup_policy in ('ignore', 'never')
1720
1721 try:
1722 self.post_process(filename, info_dict)
1723 except (PostProcessingError) as err:
1724 self.report_error('postprocessing: %s' % str(err))
1725 return
1726 self.record_download_archive(info_dict)
1727
1728 def download(self, url_list):
1729 """Download a given list of URLs."""
1730 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1731 if (len(url_list) > 1 and
1732 '%' not in outtmpl and
1733 self.params.get('max_downloads') != 1):
1734 raise SameFileError(outtmpl)
1735
1736 for url in url_list:
1737 try:
1738 # It also downloads the videos
1739 res = self.extract_info(
1740 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1741 except UnavailableVideoError:
1742 self.report_error('unable to download video')
1743 except MaxDownloadsReached:
1744 self.to_screen('[info] Maximum number of downloaded files reached.')
1745 raise
1746 else:
1747 if self.params.get('dump_single_json', False):
1748 self.to_stdout(json.dumps(res))
1749
1750 return self._download_retcode
1751
1752 def download_with_info_file(self, info_filename):
1753 with contextlib.closing(fileinput.FileInput(
1754 [info_filename], mode='r',
1755 openhook=fileinput.hook_encoded('utf-8'))) as f:
1756 # FileInput doesn't have a read method, we can't call json.load
1757 info = self.filter_requested_info(json.loads('\n'.join(f)))
1758 try:
1759 self.process_ie_result(info, download=True)
1760 except DownloadError:
1761 webpage_url = info.get('webpage_url')
1762 if webpage_url is not None:
1763 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1764 return self.download([webpage_url])
1765 else:
1766 raise
1767 return self._download_retcode
1768
1769 @staticmethod
1770 def filter_requested_info(info_dict):
1771 return dict(
1772 (k, v) for k, v in info_dict.items()
1773 if k not in ['requested_formats', 'requested_subtitles'])
1774
1775 def post_process(self, filename, ie_info):
1776 """Run all the postprocessors on the given file."""
1777 info = dict(ie_info)
1778 info['filepath'] = filename
1779 pps_chain = []
1780 if ie_info.get('__postprocessors') is not None:
1781 pps_chain.extend(ie_info['__postprocessors'])
1782 pps_chain.extend(self._pps)
1783 for pp in pps_chain:
1784 files_to_delete = []
1785 try:
1786 files_to_delete, info = pp.run(info)
1787 except PostProcessingError as e:
1788 self.report_error(e.msg)
1789 if files_to_delete and not self.params.get('keepvideo', False):
1790 for old_filename in files_to_delete:
1791 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1792 try:
1793 os.remove(encodeFilename(old_filename))
1794 except (IOError, OSError):
1795 self.report_warning('Unable to remove downloaded original file')
1796
1797 def _make_archive_id(self, info_dict):
1798 # Future-proof against any change in case
1799 # and backwards compatibility with prior versions
1800 extractor = info_dict.get('extractor_key')
1801 if extractor is None:
1802 if 'id' in info_dict:
1803 extractor = info_dict.get('ie_key') # key in a playlist
1804 if extractor is None:
1805 return None # Incomplete video information
1806 return extractor.lower() + ' ' + info_dict['id']
1807
1808 def in_download_archive(self, info_dict):
1809 fn = self.params.get('download_archive')
1810 if fn is None:
1811 return False
1812
1813 vid_id = self._make_archive_id(info_dict)
1814 if vid_id is None:
1815 return False # Incomplete video information
1816
1817 try:
1818 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1819 for line in archive_file:
1820 if line.strip() == vid_id:
1821 return True
1822 except IOError as ioe:
1823 if ioe.errno != errno.ENOENT:
1824 raise
1825 return False
1826
1827 def record_download_archive(self, info_dict):
1828 fn = self.params.get('download_archive')
1829 if fn is None:
1830 return
1831 vid_id = self._make_archive_id(info_dict)
1832 assert vid_id
1833 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1834 archive_file.write(vid_id + '\n')
1835
1836 @staticmethod
1837 def format_resolution(format, default='unknown'):
1838 if format.get('vcodec') == 'none':
1839 return 'audio only'
1840 if format.get('resolution') is not None:
1841 return format['resolution']
1842 if format.get('height') is not None:
1843 if format.get('width') is not None:
1844 res = '%sx%s' % (format['width'], format['height'])
1845 else:
1846 res = '%sp' % format['height']
1847 elif format.get('width') is not None:
1848 res = '%dx?' % format['width']
1849 else:
1850 res = default
1851 return res
1852
1853 def _format_note(self, fdict):
1854 res = ''
1855 if fdict.get('ext') in ['f4f', 'f4m']:
1856 res += '(unsupported) '
1857 if fdict.get('language'):
1858 if res:
1859 res += ' '
1860 res += '[%s] ' % fdict['language']
1861 if fdict.get('format_note') is not None:
1862 res += fdict['format_note'] + ' '
1863 if fdict.get('tbr') is not None:
1864 res += '%4dk ' % fdict['tbr']
1865 if fdict.get('container') is not None:
1866 if res:
1867 res += ', '
1868 res += '%s container' % fdict['container']
1869 if (fdict.get('vcodec') is not None and
1870 fdict.get('vcodec') != 'none'):
1871 if res:
1872 res += ', '
1873 res += fdict['vcodec']
1874 if fdict.get('vbr') is not None:
1875 res += '@'
1876 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1877 res += 'video@'
1878 if fdict.get('vbr') is not None:
1879 res += '%4dk' % fdict['vbr']
1880 if fdict.get('fps') is not None:
1881 if res:
1882 res += ', '
1883 res += '%sfps' % fdict['fps']
1884 if fdict.get('acodec') is not None:
1885 if res:
1886 res += ', '
1887 if fdict['acodec'] == 'none':
1888 res += 'video only'
1889 else:
1890 res += '%-5s' % fdict['acodec']
1891 elif fdict.get('abr') is not None:
1892 if res:
1893 res += ', '
1894 res += 'audio'
1895 if fdict.get('abr') is not None:
1896 res += '@%3dk' % fdict['abr']
1897 if fdict.get('asr') is not None:
1898 res += ' (%5dHz)' % fdict['asr']
1899 if fdict.get('filesize') is not None:
1900 if res:
1901 res += ', '
1902 res += format_bytes(fdict['filesize'])
1903 elif fdict.get('filesize_approx') is not None:
1904 if res:
1905 res += ', '
1906 res += '~' + format_bytes(fdict['filesize_approx'])
1907 return res
1908
1909 def list_formats(self, info_dict):
1910 formats = info_dict.get('formats', [info_dict])
1911 table = [
1912 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1913 for f in formats
1914 if f.get('preference') is None or f['preference'] >= -1000]
1915 if len(formats) > 1:
1916 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1917
1918 header_line = ['format code', 'extension', 'resolution', 'note']
1919 self.to_screen(
1920 '[info] Available formats for %s:\n%s' %
1921 (info_dict['id'], render_table(header_line, table)))
1922
1923 def list_thumbnails(self, info_dict):
1924 thumbnails = info_dict.get('thumbnails')
1925 if not thumbnails:
1926 self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
1927 return
1928
1929 self.to_screen(
1930 '[info] Thumbnails for %s:' % info_dict['id'])
1931 self.to_screen(render_table(
1932 ['ID', 'width', 'height', 'URL'],
1933 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1934
1935 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1936 if not subtitles:
1937 self.to_screen('%s has no %s' % (video_id, name))
1938 return
1939 self.to_screen(
1940 'Available %s for %s:' % (name, video_id))
1941 self.to_screen(render_table(
1942 ['Language', 'formats'],
1943 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1944 for lang, formats in subtitles.items()]))
1945
1946 def urlopen(self, req):
1947 """ Start an HTTP download """
1948 if isinstance(req, compat_basestring):
1949 req = sanitized_Request(req)
1950 return self._opener.open(req, timeout=self._socket_timeout)
1951
1952 def print_debug_header(self):
1953 if not self.params.get('verbose'):
1954 return
1955
1956 if type('') is not compat_str:
1957 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1958 self.report_warning(
1959 'Your Python is broken! Update to a newer and supported version')
1960
1961 stdout_encoding = getattr(
1962 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1963 encoding_str = (
1964 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1965 locale.getpreferredencoding(),
1966 sys.getfilesystemencoding(),
1967 stdout_encoding,
1968 self.get_encoding()))
1969 write_string(encoding_str, encoding=None)
1970
1971 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1972 if _LAZY_LOADER:
1973 self._write_string('[debug] Lazy loading extractors enabled' + '\n')
1974 try:
1975 sp = subprocess.Popen(
1976 ['git', 'rev-parse', '--short', 'HEAD'],
1977 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1978 cwd=os.path.dirname(os.path.abspath(__file__)))
1979 out, err = sp.communicate()
1980 out = out.decode().strip()
1981 if re.match('[0-9a-f]+', out):
1982 self._write_string('[debug] Git HEAD: ' + out + '\n')
1983 except Exception:
1984 try:
1985 sys.exc_clear()
1986 except Exception:
1987 pass
1988 self._write_string('[debug] Python version %s - %s\n' % (
1989 platform.python_version(), platform_name()))
1990
1991 exe_versions = FFmpegPostProcessor.get_versions(self)
1992 exe_versions['rtmpdump'] = rtmpdump_version()
1993 exe_str = ', '.join(
1994 '%s %s' % (exe, v)
1995 for exe, v in sorted(exe_versions.items())
1996 if v
1997 )
1998 if not exe_str:
1999 exe_str = 'none'
2000 self._write_string('[debug] exe versions: %s\n' % exe_str)
2001
2002 proxy_map = {}
2003 for handler in self._opener.handlers:
2004 if hasattr(handler, 'proxies'):
2005 proxy_map.update(handler.proxies)
2006 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2007
2008 if self.params.get('call_home', False):
2009 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2010 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2011 latest_version = self.urlopen(
2012 'https://yt-dl.org/latest/version').read().decode('utf-8')
2013 if version_tuple(latest_version) > version_tuple(__version__):
2014 self.report_warning(
2015 'You are using an outdated version (newest version: %s)! '
2016 'See https://yt-dl.org/update if you need help updating.' %
2017 latest_version)
2018
2019 def _setup_opener(self):
2020 timeout_val = self.params.get('socket_timeout')
2021 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2022
2023 opts_cookiefile = self.params.get('cookiefile')
2024 opts_proxy = self.params.get('proxy')
2025
2026 if opts_cookiefile is None:
2027 self.cookiejar = compat_cookiejar.CookieJar()
2028 else:
2029 opts_cookiefile = compat_expanduser(opts_cookiefile)
2030 self.cookiejar = compat_cookiejar.MozillaCookieJar(
2031 opts_cookiefile)
2032 if os.access(opts_cookiefile, os.R_OK):
2033 self.cookiejar.load()
2034
2035 cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2036 if opts_proxy is not None:
2037 if opts_proxy == '':
2038 proxies = {}
2039 else:
2040 proxies = {'http': opts_proxy, 'https': opts_proxy}
2041 else:
2042 proxies = compat_urllib_request.getproxies()
2043 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2044 if 'http' in proxies and 'https' not in proxies:
2045 proxies['https'] = proxies['http']
2046 proxy_handler = PerRequestProxyHandler(proxies)
2047
2048 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2049 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2050 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2051 data_handler = compat_urllib_request_DataHandler()
2052
2053 # When passing our own FileHandler instance, build_opener won't add the
2054 # default FileHandler and allows us to disable the file protocol, which
2055 # can be used for malicious purposes (see
2056 # https://github.com/rg3/youtube-dl/issues/8227)
2057 file_handler = compat_urllib_request.FileHandler()
2058
2059 def file_open(*args, **kwargs):
2060 raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2061 file_handler.file_open = file_open
2062
2063 opener = compat_urllib_request.build_opener(
2064 proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2065
2066 # Delete the default user-agent header, which would otherwise apply in
2067 # cases where our custom HTTP handler doesn't come into play
2068 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2069 opener.addheaders = []
2070 self._opener = opener
2071
2072 def encode(self, s):
2073 if isinstance(s, bytes):
2074 return s # Already encoded
2075
2076 try:
2077 return s.encode(self.get_encoding())
2078 except UnicodeEncodeError as err:
2079 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2080 raise
2081
2082 def get_encoding(self):
2083 encoding = self.params.get('encoding')
2084 if encoding is None:
2085 encoding = preferredencoding()
2086 return encoding
2087
2088 def _write_thumbnails(self, info_dict, filename):
2089 if self.params.get('writethumbnail', False):
2090 thumbnails = info_dict.get('thumbnails')
2091 if thumbnails:
2092 thumbnails = [thumbnails[-1]]
2093 elif self.params.get('write_all_thumbnails', False):
2094 thumbnails = info_dict.get('thumbnails')
2095 else:
2096 return
2097
2098 if not thumbnails:
2099 # No thumbnails present, so return immediately
2100 return
2101
2102 for t in thumbnails:
2103 thumb_ext = determine_ext(t['url'], 'jpg')
2104 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2105 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2106 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2107
2108 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2109 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2110 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2111 else:
2112 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2113 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2114 try:
2115 uf = self.urlopen(t['url'])
2116 with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2117 shutil.copyfileobj(uf, thumbf)
2118 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2119 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2120 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2121 self.report_warning('Unable to download thumbnail "%s": %s' %
2122 (t['url'], error_to_compat_str(err)))