]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py
9a8c7da05172e342959d2ec5ffa481ab951fa763
[youtubedl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import absolute_import, unicode_literals
5
6 import collections
7 import contextlib
8 import datetime
9 import errno
10 import fileinput
11 import io
12 import itertools
13 import json
14 import locale
15 import operator
16 import os
17 import platform
18 import re
19 import shutil
20 import subprocess
21 import socket
22 import sys
23 import time
24 import tokenize
25 import traceback
26
27 if os.name == 'nt':
28 import ctypes
29
30 from .compat import (
31 compat_basestring,
32 compat_cookiejar,
33 compat_expanduser,
34 compat_get_terminal_size,
35 compat_http_client,
36 compat_kwargs,
37 compat_str,
38 compat_tokenize_tokenize,
39 compat_urllib_error,
40 compat_urllib_request,
41 compat_urllib_request_DataHandler,
42 )
43 from .utils import (
44 ContentTooShortError,
45 date_from_str,
46 DateRange,
47 DEFAULT_OUTTMPL,
48 determine_ext,
49 DownloadError,
50 encodeFilename,
51 ExtractorError,
52 format_bytes,
53 formatSeconds,
54 locked_file,
55 make_HTTPS_handler,
56 MaxDownloadsReached,
57 PagedList,
58 parse_filesize,
59 PerRequestProxyHandler,
60 PostProcessingError,
61 platform_name,
62 preferredencoding,
63 render_table,
64 SameFileError,
65 sanitize_filename,
66 sanitize_path,
67 sanitized_Request,
68 std_headers,
69 subtitles_filename,
70 UnavailableVideoError,
71 url_basename,
72 version_tuple,
73 write_json_file,
74 write_string,
75 YoutubeDLCookieProcessor,
76 YoutubeDLHandler,
77 prepend_extension,
78 replace_extension,
79 args_to_str,
80 age_restricted,
81 )
82 from .cache import Cache
83 from .extractor import get_info_extractor, gen_extractors
84 from .downloader import get_suitable_downloader
85 from .downloader.rtmp import rtmpdump_version
86 from .postprocessor import (
87 FFmpegFixupM4aPP,
88 FFmpegFixupStretchedPP,
89 FFmpegMergerPP,
90 FFmpegPostProcessor,
91 get_postprocessor,
92 )
93 from .version import __version__
94
95
96 class YoutubeDL(object):
97 """YoutubeDL class.
98
99 YoutubeDL objects are the ones responsible of downloading the
100 actual video file and writing it to disk if the user has requested
101 it, among some other tasks. In most cases there should be one per
102 program. As, given a video URL, the downloader doesn't know how to
103 extract all the needed information, task that InfoExtractors do, it
104 has to pass the URL to one of them.
105
106 For this, YoutubeDL objects have a method that allows
107 InfoExtractors to be registered in a given order. When it is passed
108 a URL, the YoutubeDL object handles it to the first InfoExtractor it
109 finds that reports being able to handle it. The InfoExtractor extracts
110 all the information about the video or videos the URL refers to, and
111 YoutubeDL process the extracted information, possibly using a File
112 Downloader to download the video.
113
114 YoutubeDL objects accept a lot of parameters. In order not to saturate
115 the object constructor with arguments, it receives a dictionary of
116 options instead. These options are available through the params
117 attribute for the InfoExtractors to use. The YoutubeDL also
118 registers itself as the downloader in charge for the InfoExtractors
119 that are added to it, so this is a "mutual registration".
120
121 Available options:
122
123 username: Username for authentication purposes.
124 password: Password for authentication purposes.
125 videopassword: Password for accessing a video.
126 usenetrc: Use netrc for authentication instead.
127 verbose: Print additional info to stdout.
128 quiet: Do not print messages to stdout.
129 no_warnings: Do not print out anything for warnings.
130 forceurl: Force printing final URL.
131 forcetitle: Force printing title.
132 forceid: Force printing ID.
133 forcethumbnail: Force printing thumbnail URL.
134 forcedescription: Force printing description.
135 forcefilename: Force printing final filename.
136 forceduration: Force printing duration.
137 forcejson: Force printing info_dict as JSON.
138 dump_single_json: Force printing the info_dict of the whole playlist
139 (or video) as a single JSON line.
140 simulate: Do not download the video files.
141 format: Video format code. See options.py for more information.
142 outtmpl: Template for output names.
143 restrictfilenames: Do not allow "&" and spaces in file names
144 ignoreerrors: Do not stop on download errors.
145 force_generic_extractor: Force downloader to use the generic extractor
146 nooverwrites: Prevent overwriting files.
147 playliststart: Playlist item to start at.
148 playlistend: Playlist item to end at.
149 playlist_items: Specific indices of playlist to download.
150 playlistreverse: Download playlist items in reverse order.
151 matchtitle: Download only matching titles.
152 rejecttitle: Reject downloads for matching titles.
153 logger: Log messages to a logging.Logger instance.
154 logtostderr: Log messages to stderr instead of stdout.
155 writedescription: Write the video description to a .description file
156 writeinfojson: Write the video description to a .info.json file
157 writeannotations: Write the video annotations to a .annotations.xml file
158 writethumbnail: Write the thumbnail image to a file
159 write_all_thumbnails: Write all thumbnail formats to files
160 writesubtitles: Write the video subtitles to a file
161 writeautomaticsub: Write the automatically generated subtitles to a file
162 allsubtitles: Downloads all the subtitles of the video
163 (requires writesubtitles or writeautomaticsub)
164 listsubtitles: Lists all available subtitles for the video
165 subtitlesformat: The format code for subtitles
166 subtitleslangs: List of languages of the subtitles to download
167 keepvideo: Keep the video file after post-processing
168 daterange: A DateRange object, download only if the upload_date is in the range.
169 skip_download: Skip the actual download of the video file
170 cachedir: Location of the cache files in the filesystem.
171 False to disable filesystem cache.
172 noplaylist: Download single video instead of a playlist if in doubt.
173 age_limit: An integer representing the user's age in years.
174 Unsuitable videos for the given age are skipped.
175 min_views: An integer representing the minimum view count the video
176 must have in order to not be skipped.
177 Videos without view count information are always
178 downloaded. None for no limit.
179 max_views: An integer representing the maximum view count.
180 Videos that are more popular than that are not
181 downloaded.
182 Videos without view count information are always
183 downloaded. None for no limit.
184 download_archive: File name of a file where all downloads are recorded.
185 Videos already present in the file are not downloaded
186 again.
187 cookiefile: File name where cookies should be read from and dumped to.
188 nocheckcertificate:Do not verify SSL certificates
189 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
190 At the moment, this is only supported by YouTube.
191 proxy: URL of the proxy server to use
192 cn_verification_proxy: URL of the proxy to use for IP address verification
193 on Chinese sites. (Experimental)
194 socket_timeout: Time to wait for unresponsive hosts, in seconds
195 bidi_workaround: Work around buggy terminals without bidirectional text
196 support, using fridibi
197 debug_printtraffic:Print out sent and received HTTP traffic
198 include_ads: Download ads as well
199 default_search: Prepend this string if an input url is not valid.
200 'auto' for elaborate guessing
201 encoding: Use this encoding instead of the system-specified.
202 extract_flat: Do not resolve URLs, return the immediate result.
203 Pass in 'in_playlist' to only show this behavior for
204 playlist items.
205 postprocessors: A list of dictionaries, each with an entry
206 * key: The name of the postprocessor. See
207 youtube_dl/postprocessor/__init__.py for a list.
208 as well as any further keyword arguments for the
209 postprocessor.
210 progress_hooks: A list of functions that get called on download
211 progress, with a dictionary with the entries
212 * status: One of "downloading", "error", or "finished".
213 Check this first and ignore unknown values.
214
215 If status is one of "downloading", or "finished", the
216 following properties may also be present:
217 * filename: The final filename (always present)
218 * tmpfilename: The filename we're currently writing to
219 * downloaded_bytes: Bytes on disk
220 * total_bytes: Size of the whole file, None if unknown
221 * total_bytes_estimate: Guess of the eventual file size,
222 None if unavailable.
223 * elapsed: The number of seconds since download started.
224 * eta: The estimated time in seconds, None if unknown
225 * speed: The download speed in bytes/second, None if
226 unknown
227 * fragment_index: The counter of the currently
228 downloaded video fragment.
229 * fragment_count: The number of fragments (= individual
230 files that will be merged)
231
232 Progress hooks are guaranteed to be called at least once
233 (with status "finished") if the download is successful.
234 merge_output_format: Extension to use when merging formats.
235 fixup: Automatically correct known faults of the file.
236 One of:
237 - "never": do nothing
238 - "warn": only emit a warning
239 - "detect_or_warn": check whether we can do anything
240 about it, warn otherwise (default)
241 source_address: (Experimental) Client-side IP address to bind to.
242 call_home: Boolean, true iff we are allowed to contact the
243 youtube-dl servers for debugging.
244 sleep_interval: Number of seconds to sleep before each download.
245 listformats: Print an overview of available video formats and exit.
246 list_thumbnails: Print a table of all thumbnails and exit.
247 match_filter: A function that gets called with the info_dict of
248 every video.
249 If it returns a message, the video is ignored.
250 If it returns None, the video is downloaded.
251 match_filter_func in utils.py is one example for this.
252 no_color: Do not emit color codes in output.
253
254 The following options determine which downloader is picked:
255 external_downloader: Executable of the external downloader to call.
256 None or unset for standard (built-in) downloader.
257 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
258
259 The following parameters are not used by YoutubeDL itself, they are used by
260 the downloader (see youtube_dl/downloader/common.py):
261 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
262 noresizebuffer, retries, continuedl, noprogress, consoletitle,
263 xattr_set_filesize, external_downloader_args.
264
265 The following options are used by the post processors:
266 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
267 otherwise prefer avconv.
268 postprocessor_args: A list of additional command-line arguments for the
269 postprocessor.
270 """
271
272 params = None
273 _ies = []
274 _pps = []
275 _download_retcode = None
276 _num_downloads = None
277 _screen_file = None
278
279 def __init__(self, params=None, auto_init=True):
280 """Create a FileDownloader object with the given options."""
281 if params is None:
282 params = {}
283 self._ies = []
284 self._ies_instances = {}
285 self._pps = []
286 self._progress_hooks = []
287 self._download_retcode = 0
288 self._num_downloads = 0
289 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
290 self._err_file = sys.stderr
291 self.params = {
292 # Default parameters
293 'nocheckcertificate': False,
294 }
295 self.params.update(params)
296 self.cache = Cache(self)
297
298 if params.get('bidi_workaround', False):
299 try:
300 import pty
301 master, slave = pty.openpty()
302 width = compat_get_terminal_size().columns
303 if width is None:
304 width_args = []
305 else:
306 width_args = ['-w', str(width)]
307 sp_kwargs = dict(
308 stdin=subprocess.PIPE,
309 stdout=slave,
310 stderr=self._err_file)
311 try:
312 self._output_process = subprocess.Popen(
313 ['bidiv'] + width_args, **sp_kwargs
314 )
315 except OSError:
316 self._output_process = subprocess.Popen(
317 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
318 self._output_channel = os.fdopen(master, 'rb')
319 except OSError as ose:
320 if ose.errno == 2:
321 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
322 else:
323 raise
324
325 if (sys.version_info >= (3,) and sys.platform != 'win32' and
326 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
327 not params.get('restrictfilenames', False)):
328 # On Python 3, the Unicode filesystem API will throw errors (#1474)
329 self.report_warning(
330 'Assuming --restrict-filenames since file system encoding '
331 'cannot encode all characters. '
332 'Set the LC_ALL environment variable to fix this.')
333 self.params['restrictfilenames'] = True
334
335 if isinstance(params.get('outtmpl'), bytes):
336 self.report_warning(
337 'Parameter outtmpl is bytes, but should be a unicode string. '
338 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
339
340 self._setup_opener()
341
342 if auto_init:
343 self.print_debug_header()
344 self.add_default_info_extractors()
345
346 for pp_def_raw in self.params.get('postprocessors', []):
347 pp_class = get_postprocessor(pp_def_raw['key'])
348 pp_def = dict(pp_def_raw)
349 del pp_def['key']
350 pp = pp_class(self, **compat_kwargs(pp_def))
351 self.add_post_processor(pp)
352
353 for ph in self.params.get('progress_hooks', []):
354 self.add_progress_hook(ph)
355
356 def warn_if_short_id(self, argv):
357 # short YouTube ID starting with dash?
358 idxs = [
359 i for i, a in enumerate(argv)
360 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
361 if idxs:
362 correct_argv = (
363 ['youtube-dl'] +
364 [a for i, a in enumerate(argv) if i not in idxs] +
365 ['--'] + [argv[i] for i in idxs]
366 )
367 self.report_warning(
368 'Long argument string detected. '
369 'Use -- to separate parameters and URLs, like this:\n%s\n' %
370 args_to_str(correct_argv))
371
372 def add_info_extractor(self, ie):
373 """Add an InfoExtractor object to the end of the list."""
374 self._ies.append(ie)
375 self._ies_instances[ie.ie_key()] = ie
376 ie.set_downloader(self)
377
378 def get_info_extractor(self, ie_key):
379 """
380 Get an instance of an IE with name ie_key, it will try to get one from
381 the _ies list, if there's no instance it will create a new one and add
382 it to the extractor list.
383 """
384 ie = self._ies_instances.get(ie_key)
385 if ie is None:
386 ie = get_info_extractor(ie_key)()
387 self.add_info_extractor(ie)
388 return ie
389
390 def add_default_info_extractors(self):
391 """
392 Add the InfoExtractors returned by gen_extractors to the end of the list
393 """
394 for ie in gen_extractors():
395 self.add_info_extractor(ie)
396
397 def add_post_processor(self, pp):
398 """Add a PostProcessor object to the end of the chain."""
399 self._pps.append(pp)
400 pp.set_downloader(self)
401
402 def add_progress_hook(self, ph):
403 """Add the progress hook (currently only for the file downloader)"""
404 self._progress_hooks.append(ph)
405
406 def _bidi_workaround(self, message):
407 if not hasattr(self, '_output_channel'):
408 return message
409
410 assert hasattr(self, '_output_process')
411 assert isinstance(message, compat_str)
412 line_count = message.count('\n') + 1
413 self._output_process.stdin.write((message + '\n').encode('utf-8'))
414 self._output_process.stdin.flush()
415 res = ''.join(self._output_channel.readline().decode('utf-8')
416 for _ in range(line_count))
417 return res[:-len('\n')]
418
419 def to_screen(self, message, skip_eol=False):
420 """Print message to stdout if not in quiet mode."""
421 return self.to_stdout(message, skip_eol, check_quiet=True)
422
423 def _write_string(self, s, out=None):
424 write_string(s, out=out, encoding=self.params.get('encoding'))
425
426 def to_stdout(self, message, skip_eol=False, check_quiet=False):
427 """Print message to stdout if not in quiet mode."""
428 if self.params.get('logger'):
429 self.params['logger'].debug(message)
430 elif not check_quiet or not self.params.get('quiet', False):
431 message = self._bidi_workaround(message)
432 terminator = ['\n', ''][skip_eol]
433 output = message + terminator
434
435 self._write_string(output, self._screen_file)
436
437 def to_stderr(self, message):
438 """Print message to stderr."""
439 assert isinstance(message, compat_str)
440 if self.params.get('logger'):
441 self.params['logger'].error(message)
442 else:
443 message = self._bidi_workaround(message)
444 output = message + '\n'
445 self._write_string(output, self._err_file)
446
447 def to_console_title(self, message):
448 if not self.params.get('consoletitle', False):
449 return
450 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
451 # c_wchar_p() might not be necessary if `message` is
452 # already of type unicode()
453 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
454 elif 'TERM' in os.environ:
455 self._write_string('\033]0;%s\007' % message, self._screen_file)
456
457 def save_console_title(self):
458 if not self.params.get('consoletitle', False):
459 return
460 if 'TERM' in os.environ:
461 # Save the title on stack
462 self._write_string('\033[22;0t', self._screen_file)
463
464 def restore_console_title(self):
465 if not self.params.get('consoletitle', False):
466 return
467 if 'TERM' in os.environ:
468 # Restore the title from stack
469 self._write_string('\033[23;0t', self._screen_file)
470
471 def __enter__(self):
472 self.save_console_title()
473 return self
474
475 def __exit__(self, *args):
476 self.restore_console_title()
477
478 if self.params.get('cookiefile') is not None:
479 self.cookiejar.save()
480
481 def trouble(self, message=None, tb=None):
482 """Determine action to take when a download problem appears.
483
484 Depending on if the downloader has been configured to ignore
485 download errors or not, this method may throw an exception or
486 not when errors are found, after printing the message.
487
488 tb, if given, is additional traceback information.
489 """
490 if message is not None:
491 self.to_stderr(message)
492 if self.params.get('verbose'):
493 if tb is None:
494 if sys.exc_info()[0]: # if .trouble has been called from an except block
495 tb = ''
496 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
497 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
498 tb += compat_str(traceback.format_exc())
499 else:
500 tb_data = traceback.format_list(traceback.extract_stack())
501 tb = ''.join(tb_data)
502 self.to_stderr(tb)
503 if not self.params.get('ignoreerrors', False):
504 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
505 exc_info = sys.exc_info()[1].exc_info
506 else:
507 exc_info = sys.exc_info()
508 raise DownloadError(message, exc_info)
509 self._download_retcode = 1
510
511 def report_warning(self, message):
512 '''
513 Print the message to stderr, it will be prefixed with 'WARNING:'
514 If stderr is a tty file the 'WARNING:' will be colored
515 '''
516 if self.params.get('logger') is not None:
517 self.params['logger'].warning(message)
518 else:
519 if self.params.get('no_warnings'):
520 return
521 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
522 _msg_header = '\033[0;33mWARNING:\033[0m'
523 else:
524 _msg_header = 'WARNING:'
525 warning_message = '%s %s' % (_msg_header, message)
526 self.to_stderr(warning_message)
527
528 def report_error(self, message, tb=None):
529 '''
530 Do the same as trouble, but prefixes the message with 'ERROR:', colored
531 in red if stderr is a tty file.
532 '''
533 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
534 _msg_header = '\033[0;31mERROR:\033[0m'
535 else:
536 _msg_header = 'ERROR:'
537 error_message = '%s %s' % (_msg_header, message)
538 self.trouble(error_message, tb)
539
540 def report_file_already_downloaded(self, file_name):
541 """Report file has already been fully downloaded."""
542 try:
543 self.to_screen('[download] %s has already been downloaded' % file_name)
544 except UnicodeEncodeError:
545 self.to_screen('[download] The file has already been downloaded')
546
547 def prepare_filename(self, info_dict):
548 """Generate the output filename."""
549 try:
550 template_dict = dict(info_dict)
551
552 template_dict['epoch'] = int(time.time())
553 autonumber_size = self.params.get('autonumber_size')
554 if autonumber_size is None:
555 autonumber_size = 5
556 autonumber_templ = '%0' + str(autonumber_size) + 'd'
557 template_dict['autonumber'] = autonumber_templ % self._num_downloads
558 if template_dict.get('playlist_index') is not None:
559 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
560 if template_dict.get('resolution') is None:
561 if template_dict.get('width') and template_dict.get('height'):
562 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
563 elif template_dict.get('height'):
564 template_dict['resolution'] = '%sp' % template_dict['height']
565 elif template_dict.get('width'):
566 template_dict['resolution'] = '?x%d' % template_dict['width']
567
568 sanitize = lambda k, v: sanitize_filename(
569 compat_str(v),
570 restricted=self.params.get('restrictfilenames'),
571 is_id=(k == 'id'))
572 template_dict = dict((k, sanitize(k, v))
573 for k, v in template_dict.items()
574 if v is not None)
575 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
576
577 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
578 tmpl = compat_expanduser(outtmpl)
579 filename = tmpl % template_dict
580 # Temporary fix for #4787
581 # 'Treat' all problem characters by passing filename through preferredencoding
582 # to workaround encoding issues with subprocess on python2 @ Windows
583 if sys.version_info < (3, 0) and sys.platform == 'win32':
584 filename = encodeFilename(filename, True).decode(preferredencoding())
585 return sanitize_path(filename)
586 except ValueError as err:
587 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
588 return None
589
590 def _match_entry(self, info_dict, incomplete):
591 """ Returns None iff the file should be downloaded """
592
593 video_title = info_dict.get('title', info_dict.get('id', 'video'))
594 if 'title' in info_dict:
595 # This can happen when we're just evaluating the playlist
596 title = info_dict['title']
597 matchtitle = self.params.get('matchtitle', False)
598 if matchtitle:
599 if not re.search(matchtitle, title, re.IGNORECASE):
600 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
601 rejecttitle = self.params.get('rejecttitle', False)
602 if rejecttitle:
603 if re.search(rejecttitle, title, re.IGNORECASE):
604 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
605 date = info_dict.get('upload_date', None)
606 if date is not None:
607 dateRange = self.params.get('daterange', DateRange())
608 if date not in dateRange:
609 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
610 view_count = info_dict.get('view_count', None)
611 if view_count is not None:
612 min_views = self.params.get('min_views')
613 if min_views is not None and view_count < min_views:
614 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
615 max_views = self.params.get('max_views')
616 if max_views is not None and view_count > max_views:
617 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
618 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
619 return 'Skipping "%s" because it is age restricted' % video_title
620 if self.in_download_archive(info_dict):
621 return '%s has already been recorded in archive' % video_title
622
623 if not incomplete:
624 match_filter = self.params.get('match_filter')
625 if match_filter is not None:
626 ret = match_filter(info_dict)
627 if ret is not None:
628 return ret
629
630 return None
631
632 @staticmethod
633 def add_extra_info(info_dict, extra_info):
634 '''Set the keys from extra_info in info dict if they are missing'''
635 for key, value in extra_info.items():
636 info_dict.setdefault(key, value)
637
638 def extract_info(self, url, download=True, ie_key=None, extra_info={},
639 process=True, force_generic_extractor=False):
640 '''
641 Returns a list with a dictionary for each video we find.
642 If 'download', also downloads the videos.
643 extra_info is a dict containing the extra values to add to each result
644 '''
645
646 if not ie_key and force_generic_extractor:
647 ie_key = 'Generic'
648
649 if ie_key:
650 ies = [self.get_info_extractor(ie_key)]
651 else:
652 ies = self._ies
653
654 for ie in ies:
655 if not ie.suitable(url):
656 continue
657
658 if not ie.working():
659 self.report_warning('The program functionality for this site has been marked as broken, '
660 'and will probably not work.')
661
662 try:
663 ie_result = ie.extract(url)
664 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
665 break
666 if isinstance(ie_result, list):
667 # Backwards compatibility: old IE result format
668 ie_result = {
669 '_type': 'compat_list',
670 'entries': ie_result,
671 }
672 self.add_default_extra_info(ie_result, ie, url)
673 if process:
674 return self.process_ie_result(ie_result, download, extra_info)
675 else:
676 return ie_result
677 except ExtractorError as de: # An error we somewhat expected
678 self.report_error(compat_str(de), de.format_traceback())
679 break
680 except MaxDownloadsReached:
681 raise
682 except Exception as e:
683 if self.params.get('ignoreerrors', False):
684 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
685 break
686 else:
687 raise
688 else:
689 self.report_error('no suitable InfoExtractor for URL %s' % url)
690
691 def add_default_extra_info(self, ie_result, ie, url):
692 self.add_extra_info(ie_result, {
693 'extractor': ie.IE_NAME,
694 'webpage_url': url,
695 'webpage_url_basename': url_basename(url),
696 'extractor_key': ie.ie_key(),
697 })
698
699 def process_ie_result(self, ie_result, download=True, extra_info={}):
700 """
701 Take the result of the ie(may be modified) and resolve all unresolved
702 references (URLs, playlist items).
703
704 It will also download the videos if 'download'.
705 Returns the resolved ie_result.
706 """
707
708 result_type = ie_result.get('_type', 'video')
709
710 if result_type in ('url', 'url_transparent'):
711 extract_flat = self.params.get('extract_flat', False)
712 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
713 extract_flat is True):
714 if self.params.get('forcejson', False):
715 self.to_stdout(json.dumps(ie_result))
716 return ie_result
717
718 if result_type == 'video':
719 self.add_extra_info(ie_result, extra_info)
720 return self.process_video_result(ie_result, download=download)
721 elif result_type == 'url':
722 # We have to add extra_info to the results because it may be
723 # contained in a playlist
724 return self.extract_info(ie_result['url'],
725 download,
726 ie_key=ie_result.get('ie_key'),
727 extra_info=extra_info)
728 elif result_type == 'url_transparent':
729 # Use the information from the embedding page
730 info = self.extract_info(
731 ie_result['url'], ie_key=ie_result.get('ie_key'),
732 extra_info=extra_info, download=False, process=False)
733
734 force_properties = dict(
735 (k, v) for k, v in ie_result.items() if v is not None)
736 for f in ('_type', 'url'):
737 if f in force_properties:
738 del force_properties[f]
739 new_result = info.copy()
740 new_result.update(force_properties)
741
742 assert new_result.get('_type') != 'url_transparent'
743
744 return self.process_ie_result(
745 new_result, download=download, extra_info=extra_info)
746 elif result_type == 'playlist' or result_type == 'multi_video':
747 # We process each entry in the playlist
748 playlist = ie_result.get('title', None) or ie_result.get('id', None)
749 self.to_screen('[download] Downloading playlist: %s' % playlist)
750
751 playlist_results = []
752
753 playliststart = self.params.get('playliststart', 1) - 1
754 playlistend = self.params.get('playlistend', None)
755 # For backwards compatibility, interpret -1 as whole list
756 if playlistend == -1:
757 playlistend = None
758
759 playlistitems_str = self.params.get('playlist_items', None)
760 playlistitems = None
761 if playlistitems_str is not None:
762 def iter_playlistitems(format):
763 for string_segment in format.split(','):
764 if '-' in string_segment:
765 start, end = string_segment.split('-')
766 for item in range(int(start), int(end) + 1):
767 yield int(item)
768 else:
769 yield int(string_segment)
770 playlistitems = iter_playlistitems(playlistitems_str)
771
772 ie_entries = ie_result['entries']
773 if isinstance(ie_entries, list):
774 n_all_entries = len(ie_entries)
775 if playlistitems:
776 entries = [
777 ie_entries[i - 1] for i in playlistitems
778 if -n_all_entries <= i - 1 < n_all_entries]
779 else:
780 entries = ie_entries[playliststart:playlistend]
781 n_entries = len(entries)
782 self.to_screen(
783 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
784 (ie_result['extractor'], playlist, n_all_entries, n_entries))
785 elif isinstance(ie_entries, PagedList):
786 if playlistitems:
787 entries = []
788 for item in playlistitems:
789 entries.extend(ie_entries.getslice(
790 item - 1, item
791 ))
792 else:
793 entries = ie_entries.getslice(
794 playliststart, playlistend)
795 n_entries = len(entries)
796 self.to_screen(
797 "[%s] playlist %s: Downloading %d videos" %
798 (ie_result['extractor'], playlist, n_entries))
799 else: # iterable
800 if playlistitems:
801 entry_list = list(ie_entries)
802 entries = [entry_list[i - 1] for i in playlistitems]
803 else:
804 entries = list(itertools.islice(
805 ie_entries, playliststart, playlistend))
806 n_entries = len(entries)
807 self.to_screen(
808 "[%s] playlist %s: Downloading %d videos" %
809 (ie_result['extractor'], playlist, n_entries))
810
811 if self.params.get('playlistreverse', False):
812 entries = entries[::-1]
813
814 for i, entry in enumerate(entries, 1):
815 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
816 extra = {
817 'n_entries': n_entries,
818 'playlist': playlist,
819 'playlist_id': ie_result.get('id'),
820 'playlist_title': ie_result.get('title'),
821 'playlist_index': i + playliststart,
822 'extractor': ie_result['extractor'],
823 'webpage_url': ie_result['webpage_url'],
824 'webpage_url_basename': url_basename(ie_result['webpage_url']),
825 'extractor_key': ie_result['extractor_key'],
826 }
827
828 reason = self._match_entry(entry, incomplete=True)
829 if reason is not None:
830 self.to_screen('[download] ' + reason)
831 continue
832
833 entry_result = self.process_ie_result(entry,
834 download=download,
835 extra_info=extra)
836 playlist_results.append(entry_result)
837 ie_result['entries'] = playlist_results
838 self.to_screen('[download] Finished downloading playlist: %s' % playlist)
839 return ie_result
840 elif result_type == 'compat_list':
841 self.report_warning(
842 'Extractor %s returned a compat_list result. '
843 'It needs to be updated.' % ie_result.get('extractor'))
844
845 def _fixup(r):
846 self.add_extra_info(
847 r,
848 {
849 'extractor': ie_result['extractor'],
850 'webpage_url': ie_result['webpage_url'],
851 'webpage_url_basename': url_basename(ie_result['webpage_url']),
852 'extractor_key': ie_result['extractor_key'],
853 }
854 )
855 return r
856 ie_result['entries'] = [
857 self.process_ie_result(_fixup(r), download, extra_info)
858 for r in ie_result['entries']
859 ]
860 return ie_result
861 else:
862 raise Exception('Invalid result type: %s' % result_type)
863
864 def _build_format_filter(self, filter_spec):
865 " Returns a function to filter the formats according to the filter_spec "
866
867 OPERATORS = {
868 '<': operator.lt,
869 '<=': operator.le,
870 '>': operator.gt,
871 '>=': operator.ge,
872 '=': operator.eq,
873 '!=': operator.ne,
874 }
875 operator_rex = re.compile(r'''(?x)\s*
876 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
877 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
878 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
879 $
880 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
881 m = operator_rex.search(filter_spec)
882 if m:
883 try:
884 comparison_value = int(m.group('value'))
885 except ValueError:
886 comparison_value = parse_filesize(m.group('value'))
887 if comparison_value is None:
888 comparison_value = parse_filesize(m.group('value') + 'B')
889 if comparison_value is None:
890 raise ValueError(
891 'Invalid value %r in format specification %r' % (
892 m.group('value'), filter_spec))
893 op = OPERATORS[m.group('op')]
894
895 if not m:
896 STR_OPERATORS = {
897 '=': operator.eq,
898 '!=': operator.ne,
899 }
900 str_operator_rex = re.compile(r'''(?x)
901 \s*(?P<key>ext|acodec|vcodec|container|protocol)
902 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
903 \s*(?P<value>[a-zA-Z0-9_-]+)
904 \s*$
905 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
906 m = str_operator_rex.search(filter_spec)
907 if m:
908 comparison_value = m.group('value')
909 op = STR_OPERATORS[m.group('op')]
910
911 if not m:
912 raise ValueError('Invalid filter specification %r' % filter_spec)
913
914 def _filter(f):
915 actual_value = f.get(m.group('key'))
916 if actual_value is None:
917 return m.group('none_inclusive')
918 return op(actual_value, comparison_value)
919 return _filter
920
921 def build_format_selector(self, format_spec):
922 def syntax_error(note, start):
923 message = (
924 'Invalid format specification: '
925 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
926 return SyntaxError(message)
927
928 PICKFIRST = 'PICKFIRST'
929 MERGE = 'MERGE'
930 SINGLE = 'SINGLE'
931 GROUP = 'GROUP'
932 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
933
934 def _parse_filter(tokens):
935 filter_parts = []
936 for type, string, start, _, _ in tokens:
937 if type == tokenize.OP and string == ']':
938 return ''.join(filter_parts)
939 else:
940 filter_parts.append(string)
941
942 def _remove_unused_ops(tokens):
943 # Remove operators that we don't use and join them with the surrounding strings
944 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
945 ALLOWED_OPS = ('/', '+', ',', '(', ')')
946 last_string, last_start, last_end, last_line = None, None, None, None
947 for type, string, start, end, line in tokens:
948 if type == tokenize.OP and string == '[':
949 if last_string:
950 yield tokenize.NAME, last_string, last_start, last_end, last_line
951 last_string = None
952 yield type, string, start, end, line
953 # everything inside brackets will be handled by _parse_filter
954 for type, string, start, end, line in tokens:
955 yield type, string, start, end, line
956 if type == tokenize.OP and string == ']':
957 break
958 elif type == tokenize.OP and string in ALLOWED_OPS:
959 if last_string:
960 yield tokenize.NAME, last_string, last_start, last_end, last_line
961 last_string = None
962 yield type, string, start, end, line
963 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
964 if not last_string:
965 last_string = string
966 last_start = start
967 last_end = end
968 else:
969 last_string += string
970 if last_string:
971 yield tokenize.NAME, last_string, last_start, last_end, last_line
972
973 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
974 selectors = []
975 current_selector = None
976 for type, string, start, _, _ in tokens:
977 # ENCODING is only defined in python 3.x
978 if type == getattr(tokenize, 'ENCODING', None):
979 continue
980 elif type in [tokenize.NAME, tokenize.NUMBER]:
981 current_selector = FormatSelector(SINGLE, string, [])
982 elif type == tokenize.OP:
983 if string == ')':
984 if not inside_group:
985 # ')' will be handled by the parentheses group
986 tokens.restore_last_token()
987 break
988 elif inside_merge and string in ['/', ',']:
989 tokens.restore_last_token()
990 break
991 elif inside_choice and string == ',':
992 tokens.restore_last_token()
993 break
994 elif string == ',':
995 if not current_selector:
996 raise syntax_error('"," must follow a format selector', start)
997 selectors.append(current_selector)
998 current_selector = None
999 elif string == '/':
1000 if not current_selector:
1001 raise syntax_error('"/" must follow a format selector', start)
1002 first_choice = current_selector
1003 second_choice = _parse_format_selection(tokens, inside_choice=True)
1004 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1005 elif string == '[':
1006 if not current_selector:
1007 current_selector = FormatSelector(SINGLE, 'best', [])
1008 format_filter = _parse_filter(tokens)
1009 current_selector.filters.append(format_filter)
1010 elif string == '(':
1011 if current_selector:
1012 raise syntax_error('Unexpected "("', start)
1013 group = _parse_format_selection(tokens, inside_group=True)
1014 current_selector = FormatSelector(GROUP, group, [])
1015 elif string == '+':
1016 video_selector = current_selector
1017 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1018 if not video_selector or not audio_selector:
1019 raise syntax_error('"+" must be between two format selectors', start)
1020 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1021 else:
1022 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1023 elif type == tokenize.ENDMARKER:
1024 break
1025 if current_selector:
1026 selectors.append(current_selector)
1027 return selectors
1028
1029 def _build_selector_function(selector):
1030 if isinstance(selector, list):
1031 fs = [_build_selector_function(s) for s in selector]
1032
1033 def selector_function(formats):
1034 for f in fs:
1035 for format in f(formats):
1036 yield format
1037 return selector_function
1038 elif selector.type == GROUP:
1039 selector_function = _build_selector_function(selector.selector)
1040 elif selector.type == PICKFIRST:
1041 fs = [_build_selector_function(s) for s in selector.selector]
1042
1043 def selector_function(formats):
1044 for f in fs:
1045 picked_formats = list(f(formats))
1046 if picked_formats:
1047 return picked_formats
1048 return []
1049 elif selector.type == SINGLE:
1050 format_spec = selector.selector
1051
1052 def selector_function(formats):
1053 formats = list(formats)
1054 if not formats:
1055 return
1056 if format_spec == 'all':
1057 for f in formats:
1058 yield f
1059 elif format_spec in ['best', 'worst', None]:
1060 format_idx = 0 if format_spec == 'worst' else -1
1061 audiovideo_formats = [
1062 f for f in formats
1063 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1064 if audiovideo_formats:
1065 yield audiovideo_formats[format_idx]
1066 # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1067 elif (all(f.get('acodec') != 'none' for f in formats) or
1068 all(f.get('vcodec') != 'none' for f in formats)):
1069 yield formats[format_idx]
1070 elif format_spec == 'bestaudio':
1071 audio_formats = [
1072 f for f in formats
1073 if f.get('vcodec') == 'none']
1074 if audio_formats:
1075 yield audio_formats[-1]
1076 elif format_spec == 'worstaudio':
1077 audio_formats = [
1078 f for f in formats
1079 if f.get('vcodec') == 'none']
1080 if audio_formats:
1081 yield audio_formats[0]
1082 elif format_spec == 'bestvideo':
1083 video_formats = [
1084 f for f in formats
1085 if f.get('acodec') == 'none']
1086 if video_formats:
1087 yield video_formats[-1]
1088 elif format_spec == 'worstvideo':
1089 video_formats = [
1090 f for f in formats
1091 if f.get('acodec') == 'none']
1092 if video_formats:
1093 yield video_formats[0]
1094 else:
1095 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1096 if format_spec in extensions:
1097 filter_f = lambda f: f['ext'] == format_spec
1098 else:
1099 filter_f = lambda f: f['format_id'] == format_spec
1100 matches = list(filter(filter_f, formats))
1101 if matches:
1102 yield matches[-1]
1103 elif selector.type == MERGE:
1104 def _merge(formats_info):
1105 format_1, format_2 = [f['format_id'] for f in formats_info]
1106 # The first format must contain the video and the
1107 # second the audio
1108 if formats_info[0].get('vcodec') == 'none':
1109 self.report_error('The first format must '
1110 'contain the video, try using '
1111 '"-f %s+%s"' % (format_2, format_1))
1112 return
1113 output_ext = (
1114 formats_info[0]['ext']
1115 if self.params.get('merge_output_format') is None
1116 else self.params['merge_output_format'])
1117 return {
1118 'requested_formats': formats_info,
1119 'format': '%s+%s' % (formats_info[0].get('format'),
1120 formats_info[1].get('format')),
1121 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1122 formats_info[1].get('format_id')),
1123 'width': formats_info[0].get('width'),
1124 'height': formats_info[0].get('height'),
1125 'resolution': formats_info[0].get('resolution'),
1126 'fps': formats_info[0].get('fps'),
1127 'vcodec': formats_info[0].get('vcodec'),
1128 'vbr': formats_info[0].get('vbr'),
1129 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1130 'acodec': formats_info[1].get('acodec'),
1131 'abr': formats_info[1].get('abr'),
1132 'ext': output_ext,
1133 }
1134 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1135
1136 def selector_function(formats):
1137 formats = list(formats)
1138 for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1139 yield _merge(pair)
1140
1141 filters = [self._build_format_filter(f) for f in selector.filters]
1142
1143 def final_selector(formats):
1144 for _filter in filters:
1145 formats = list(filter(_filter, formats))
1146 return selector_function(formats)
1147 return final_selector
1148
1149 stream = io.BytesIO(format_spec.encode('utf-8'))
1150 try:
1151 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1152 except tokenize.TokenError:
1153 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1154
1155 class TokenIterator(object):
1156 def __init__(self, tokens):
1157 self.tokens = tokens
1158 self.counter = 0
1159
1160 def __iter__(self):
1161 return self
1162
1163 def __next__(self):
1164 if self.counter >= len(self.tokens):
1165 raise StopIteration()
1166 value = self.tokens[self.counter]
1167 self.counter += 1
1168 return value
1169
1170 next = __next__
1171
1172 def restore_last_token(self):
1173 self.counter -= 1
1174
1175 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1176 return _build_selector_function(parsed_selector)
1177
1178 def _calc_headers(self, info_dict):
1179 res = std_headers.copy()
1180
1181 add_headers = info_dict.get('http_headers')
1182 if add_headers:
1183 res.update(add_headers)
1184
1185 cookies = self._calc_cookies(info_dict)
1186 if cookies:
1187 res['Cookie'] = cookies
1188
1189 return res
1190
1191 def _calc_cookies(self, info_dict):
1192 pr = sanitized_Request(info_dict['url'])
1193 self.cookiejar.add_cookie_header(pr)
1194 return pr.get_header('Cookie')
1195
1196 def process_video_result(self, info_dict, download=True):
1197 assert info_dict.get('_type', 'video') == 'video'
1198
1199 if 'id' not in info_dict:
1200 raise ExtractorError('Missing "id" field in extractor result')
1201 if 'title' not in info_dict:
1202 raise ExtractorError('Missing "title" field in extractor result')
1203
1204 if 'playlist' not in info_dict:
1205 # It isn't part of a playlist
1206 info_dict['playlist'] = None
1207 info_dict['playlist_index'] = None
1208
1209 thumbnails = info_dict.get('thumbnails')
1210 if thumbnails is None:
1211 thumbnail = info_dict.get('thumbnail')
1212 if thumbnail:
1213 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1214 if thumbnails:
1215 thumbnails.sort(key=lambda t: (
1216 t.get('preference'), t.get('width'), t.get('height'),
1217 t.get('id'), t.get('url')))
1218 for i, t in enumerate(thumbnails):
1219 if t.get('width') and t.get('height'):
1220 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1221 if t.get('id') is None:
1222 t['id'] = '%d' % i
1223
1224 if thumbnails and 'thumbnail' not in info_dict:
1225 info_dict['thumbnail'] = thumbnails[-1]['url']
1226
1227 if 'display_id' not in info_dict and 'id' in info_dict:
1228 info_dict['display_id'] = info_dict['id']
1229
1230 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1231 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1232 # see http://bugs.python.org/issue1646728)
1233 try:
1234 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1235 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1236 except (ValueError, OverflowError, OSError):
1237 pass
1238
1239 subtitles = info_dict.get('subtitles')
1240 if subtitles:
1241 for _, subtitle in subtitles.items():
1242 for subtitle_format in subtitle:
1243 if 'ext' not in subtitle_format:
1244 subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1245
1246 if self.params.get('listsubtitles', False):
1247 if 'automatic_captions' in info_dict:
1248 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1249 self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1250 return
1251 info_dict['requested_subtitles'] = self.process_subtitles(
1252 info_dict['id'], subtitles,
1253 info_dict.get('automatic_captions'))
1254
1255 # We now pick which formats have to be downloaded
1256 if info_dict.get('formats') is None:
1257 # There's only one format available
1258 formats = [info_dict]
1259 else:
1260 formats = info_dict['formats']
1261
1262 if not formats:
1263 raise ExtractorError('No video formats found!')
1264
1265 formats_dict = {}
1266
1267 # We check that all the formats have the format and format_id fields
1268 for i, format in enumerate(formats):
1269 if 'url' not in format:
1270 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1271
1272 if format.get('format_id') is None:
1273 format['format_id'] = compat_str(i)
1274 format_id = format['format_id']
1275 if format_id not in formats_dict:
1276 formats_dict[format_id] = []
1277 formats_dict[format_id].append(format)
1278
1279 # Make sure all formats have unique format_id
1280 for format_id, ambiguous_formats in formats_dict.items():
1281 if len(ambiguous_formats) > 1:
1282 for i, format in enumerate(ambiguous_formats):
1283 format['format_id'] = '%s-%d' % (format_id, i)
1284
1285 for i, format in enumerate(formats):
1286 if format.get('format') is None:
1287 format['format'] = '{id} - {res}{note}'.format(
1288 id=format['format_id'],
1289 res=self.format_resolution(format),
1290 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1291 )
1292 # Automatically determine file extension if missing
1293 if 'ext' not in format:
1294 format['ext'] = determine_ext(format['url']).lower()
1295 # Add HTTP headers, so that external programs can use them from the
1296 # json output
1297 full_format_info = info_dict.copy()
1298 full_format_info.update(format)
1299 format['http_headers'] = self._calc_headers(full_format_info)
1300
1301 # TODO Central sorting goes here
1302
1303 if formats[0] is not info_dict:
1304 # only set the 'formats' fields if the original info_dict list them
1305 # otherwise we end up with a circular reference, the first (and unique)
1306 # element in the 'formats' field in info_dict is info_dict itself,
1307 # wich can't be exported to json
1308 info_dict['formats'] = formats
1309 if self.params.get('listformats'):
1310 self.list_formats(info_dict)
1311 return
1312 if self.params.get('list_thumbnails'):
1313 self.list_thumbnails(info_dict)
1314 return
1315
1316 req_format = self.params.get('format')
1317 if req_format is None:
1318 req_format_list = []
1319 if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1320 info_dict['extractor'] in ['youtube', 'ted'] and
1321 not info_dict.get('is_live')):
1322 merger = FFmpegMergerPP(self)
1323 if merger.available and merger.can_merge():
1324 req_format_list.append('bestvideo+bestaudio')
1325 req_format_list.append('best')
1326 req_format = '/'.join(req_format_list)
1327 format_selector = self.build_format_selector(req_format)
1328 formats_to_download = list(format_selector(formats))
1329 if not formats_to_download:
1330 raise ExtractorError('requested format not available',
1331 expected=True)
1332
1333 if download:
1334 if len(formats_to_download) > 1:
1335 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1336 for format in formats_to_download:
1337 new_info = dict(info_dict)
1338 new_info.update(format)
1339 self.process_info(new_info)
1340 # We update the info dict with the best quality format (backwards compatibility)
1341 info_dict.update(formats_to_download[-1])
1342 return info_dict
1343
1344 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1345 """Select the requested subtitles and their format"""
1346 available_subs = {}
1347 if normal_subtitles and self.params.get('writesubtitles'):
1348 available_subs.update(normal_subtitles)
1349 if automatic_captions and self.params.get('writeautomaticsub'):
1350 for lang, cap_info in automatic_captions.items():
1351 if lang not in available_subs:
1352 available_subs[lang] = cap_info
1353
1354 if (not self.params.get('writesubtitles') and not
1355 self.params.get('writeautomaticsub') or not
1356 available_subs):
1357 return None
1358
1359 if self.params.get('allsubtitles', False):
1360 requested_langs = available_subs.keys()
1361 else:
1362 if self.params.get('subtitleslangs', False):
1363 requested_langs = self.params.get('subtitleslangs')
1364 elif 'en' in available_subs:
1365 requested_langs = ['en']
1366 else:
1367 requested_langs = [list(available_subs.keys())[0]]
1368
1369 formats_query = self.params.get('subtitlesformat', 'best')
1370 formats_preference = formats_query.split('/') if formats_query else []
1371 subs = {}
1372 for lang in requested_langs:
1373 formats = available_subs.get(lang)
1374 if formats is None:
1375 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1376 continue
1377 for ext in formats_preference:
1378 if ext == 'best':
1379 f = formats[-1]
1380 break
1381 matches = list(filter(lambda f: f['ext'] == ext, formats))
1382 if matches:
1383 f = matches[-1]
1384 break
1385 else:
1386 f = formats[-1]
1387 self.report_warning(
1388 'No subtitle format found matching "%s" for language %s, '
1389 'using %s' % (formats_query, lang, f['ext']))
1390 subs[lang] = f
1391 return subs
1392
1393 def process_info(self, info_dict):
1394 """Process a single resolved IE result."""
1395
1396 assert info_dict.get('_type', 'video') == 'video'
1397
1398 max_downloads = self.params.get('max_downloads')
1399 if max_downloads is not None:
1400 if self._num_downloads >= int(max_downloads):
1401 raise MaxDownloadsReached()
1402
1403 info_dict['fulltitle'] = info_dict['title']
1404 if len(info_dict['title']) > 200:
1405 info_dict['title'] = info_dict['title'][:197] + '...'
1406
1407 if 'format' not in info_dict:
1408 info_dict['format'] = info_dict['ext']
1409
1410 reason = self._match_entry(info_dict, incomplete=False)
1411 if reason is not None:
1412 self.to_screen('[download] ' + reason)
1413 return
1414
1415 self._num_downloads += 1
1416
1417 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1418
1419 # Forced printings
1420 if self.params.get('forcetitle', False):
1421 self.to_stdout(info_dict['fulltitle'])
1422 if self.params.get('forceid', False):
1423 self.to_stdout(info_dict['id'])
1424 if self.params.get('forceurl', False):
1425 if info_dict.get('requested_formats') is not None:
1426 for f in info_dict['requested_formats']:
1427 self.to_stdout(f['url'] + f.get('play_path', ''))
1428 else:
1429 # For RTMP URLs, also include the playpath
1430 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1431 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1432 self.to_stdout(info_dict['thumbnail'])
1433 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1434 self.to_stdout(info_dict['description'])
1435 if self.params.get('forcefilename', False) and filename is not None:
1436 self.to_stdout(filename)
1437 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1438 self.to_stdout(formatSeconds(info_dict['duration']))
1439 if self.params.get('forceformat', False):
1440 self.to_stdout(info_dict['format'])
1441 if self.params.get('forcejson', False):
1442 self.to_stdout(json.dumps(info_dict))
1443
1444 # Do nothing else if in simulate mode
1445 if self.params.get('simulate', False):
1446 return
1447
1448 if filename is None:
1449 return
1450
1451 try:
1452 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1453 if dn and not os.path.exists(dn):
1454 os.makedirs(dn)
1455 except (OSError, IOError) as err:
1456 self.report_error('unable to create directory ' + compat_str(err))
1457 return
1458
1459 if self.params.get('writedescription', False):
1460 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1461 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1462 self.to_screen('[info] Video description is already present')
1463 elif info_dict.get('description') is None:
1464 self.report_warning('There\'s no description to write.')
1465 else:
1466 try:
1467 self.to_screen('[info] Writing video description to: ' + descfn)
1468 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1469 descfile.write(info_dict['description'])
1470 except (OSError, IOError):
1471 self.report_error('Cannot write description file ' + descfn)
1472 return
1473
1474 if self.params.get('writeannotations', False):
1475 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1476 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1477 self.to_screen('[info] Video annotations are already present')
1478 else:
1479 try:
1480 self.to_screen('[info] Writing video annotations to: ' + annofn)
1481 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1482 annofile.write(info_dict['annotations'])
1483 except (KeyError, TypeError):
1484 self.report_warning('There are no annotations to write.')
1485 except (OSError, IOError):
1486 self.report_error('Cannot write annotations file: ' + annofn)
1487 return
1488
1489 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1490 self.params.get('writeautomaticsub')])
1491
1492 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1493 # subtitles download errors are already managed as troubles in relevant IE
1494 # that way it will silently go on when used with unsupporting IE
1495 subtitles = info_dict['requested_subtitles']
1496 ie = self.get_info_extractor(info_dict['extractor_key'])
1497 for sub_lang, sub_info in subtitles.items():
1498 sub_format = sub_info['ext']
1499 if sub_info.get('data') is not None:
1500 sub_data = sub_info['data']
1501 else:
1502 try:
1503 sub_data = ie._download_webpage(
1504 sub_info['url'], info_dict['id'], note=False)
1505 except ExtractorError as err:
1506 self.report_warning('Unable to download subtitle for "%s": %s' %
1507 (sub_lang, compat_str(err.cause)))
1508 continue
1509 try:
1510 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1511 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1512 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1513 else:
1514 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1515 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1516 subfile.write(sub_data)
1517 except (OSError, IOError):
1518 self.report_error('Cannot write subtitles file ' + sub_filename)
1519 return
1520
1521 if self.params.get('writeinfojson', False):
1522 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1523 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1524 self.to_screen('[info] Video description metadata is already present')
1525 else:
1526 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1527 try:
1528 write_json_file(self.filter_requested_info(info_dict), infofn)
1529 except (OSError, IOError):
1530 self.report_error('Cannot write metadata to JSON file ' + infofn)
1531 return
1532
1533 self._write_thumbnails(info_dict, filename)
1534
1535 if not self.params.get('skip_download', False):
1536 try:
1537 def dl(name, info):
1538 fd = get_suitable_downloader(info, self.params)(self, self.params)
1539 for ph in self._progress_hooks:
1540 fd.add_progress_hook(ph)
1541 if self.params.get('verbose'):
1542 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1543 return fd.download(name, info)
1544
1545 if info_dict.get('requested_formats') is not None:
1546 downloaded = []
1547 success = True
1548 merger = FFmpegMergerPP(self)
1549 if not merger.available:
1550 postprocessors = []
1551 self.report_warning('You have requested multiple '
1552 'formats but ffmpeg or avconv are not installed.'
1553 ' The formats won\'t be merged.')
1554 else:
1555 postprocessors = [merger]
1556
1557 def compatible_formats(formats):
1558 video, audio = formats
1559 # Check extension
1560 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1561 if video_ext and audio_ext:
1562 COMPATIBLE_EXTS = (
1563 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1564 ('webm')
1565 )
1566 for exts in COMPATIBLE_EXTS:
1567 if video_ext in exts and audio_ext in exts:
1568 return True
1569 # TODO: Check acodec/vcodec
1570 return False
1571
1572 filename_real_ext = os.path.splitext(filename)[1][1:]
1573 filename_wo_ext = (
1574 os.path.splitext(filename)[0]
1575 if filename_real_ext == info_dict['ext']
1576 else filename)
1577 requested_formats = info_dict['requested_formats']
1578 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1579 info_dict['ext'] = 'mkv'
1580 self.report_warning(
1581 'Requested formats are incompatible for merge and will be merged into mkv.')
1582 # Ensure filename always has a correct extension for successful merge
1583 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1584 if os.path.exists(encodeFilename(filename)):
1585 self.to_screen(
1586 '[download] %s has already been downloaded and '
1587 'merged' % filename)
1588 else:
1589 for f in requested_formats:
1590 new_info = dict(info_dict)
1591 new_info.update(f)
1592 fname = self.prepare_filename(new_info)
1593 fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1594 downloaded.append(fname)
1595 partial_success = dl(fname, new_info)
1596 success = success and partial_success
1597 info_dict['__postprocessors'] = postprocessors
1598 info_dict['__files_to_merge'] = downloaded
1599 else:
1600 # Just a single file
1601 success = dl(filename, info_dict)
1602 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1603 self.report_error('unable to download video data: %s' % str(err))
1604 return
1605 except (OSError, IOError) as err:
1606 raise UnavailableVideoError(err)
1607 except (ContentTooShortError, ) as err:
1608 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1609 return
1610
1611 if success:
1612 # Fixup content
1613 fixup_policy = self.params.get('fixup')
1614 if fixup_policy is None:
1615 fixup_policy = 'detect_or_warn'
1616
1617 stretched_ratio = info_dict.get('stretched_ratio')
1618 if stretched_ratio is not None and stretched_ratio != 1:
1619 if fixup_policy == 'warn':
1620 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1621 info_dict['id'], stretched_ratio))
1622 elif fixup_policy == 'detect_or_warn':
1623 stretched_pp = FFmpegFixupStretchedPP(self)
1624 if stretched_pp.available:
1625 info_dict.setdefault('__postprocessors', [])
1626 info_dict['__postprocessors'].append(stretched_pp)
1627 else:
1628 self.report_warning(
1629 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1630 info_dict['id'], stretched_ratio))
1631 else:
1632 assert fixup_policy in ('ignore', 'never')
1633
1634 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1635 if fixup_policy == 'warn':
1636 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1637 info_dict['id']))
1638 elif fixup_policy == 'detect_or_warn':
1639 fixup_pp = FFmpegFixupM4aPP(self)
1640 if fixup_pp.available:
1641 info_dict.setdefault('__postprocessors', [])
1642 info_dict['__postprocessors'].append(fixup_pp)
1643 else:
1644 self.report_warning(
1645 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1646 info_dict['id']))
1647 else:
1648 assert fixup_policy in ('ignore', 'never')
1649
1650 try:
1651 self.post_process(filename, info_dict)
1652 except (PostProcessingError) as err:
1653 self.report_error('postprocessing: %s' % str(err))
1654 return
1655 self.record_download_archive(info_dict)
1656
1657 def download(self, url_list):
1658 """Download a given list of URLs."""
1659 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1660 if (len(url_list) > 1 and
1661 '%' not in outtmpl and
1662 self.params.get('max_downloads') != 1):
1663 raise SameFileError(outtmpl)
1664
1665 for url in url_list:
1666 try:
1667 # It also downloads the videos
1668 res = self.extract_info(
1669 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1670 except UnavailableVideoError:
1671 self.report_error('unable to download video')
1672 except MaxDownloadsReached:
1673 self.to_screen('[info] Maximum number of downloaded files reached.')
1674 raise
1675 else:
1676 if self.params.get('dump_single_json', False):
1677 self.to_stdout(json.dumps(res))
1678
1679 return self._download_retcode
1680
1681 def download_with_info_file(self, info_filename):
1682 with contextlib.closing(fileinput.FileInput(
1683 [info_filename], mode='r',
1684 openhook=fileinput.hook_encoded('utf-8'))) as f:
1685 # FileInput doesn't have a read method, we can't call json.load
1686 info = self.filter_requested_info(json.loads('\n'.join(f)))
1687 try:
1688 self.process_ie_result(info, download=True)
1689 except DownloadError:
1690 webpage_url = info.get('webpage_url')
1691 if webpage_url is not None:
1692 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1693 return self.download([webpage_url])
1694 else:
1695 raise
1696 return self._download_retcode
1697
1698 @staticmethod
1699 def filter_requested_info(info_dict):
1700 return dict(
1701 (k, v) for k, v in info_dict.items()
1702 if k not in ['requested_formats', 'requested_subtitles'])
1703
1704 def post_process(self, filename, ie_info):
1705 """Run all the postprocessors on the given file."""
1706 info = dict(ie_info)
1707 info['filepath'] = filename
1708 pps_chain = []
1709 if ie_info.get('__postprocessors') is not None:
1710 pps_chain.extend(ie_info['__postprocessors'])
1711 pps_chain.extend(self._pps)
1712 for pp in pps_chain:
1713 files_to_delete = []
1714 try:
1715 files_to_delete, info = pp.run(info)
1716 except PostProcessingError as e:
1717 self.report_error(e.msg)
1718 if files_to_delete and not self.params.get('keepvideo', False):
1719 for old_filename in files_to_delete:
1720 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1721 try:
1722 os.remove(encodeFilename(old_filename))
1723 except (IOError, OSError):
1724 self.report_warning('Unable to remove downloaded original file')
1725
1726 def _make_archive_id(self, info_dict):
1727 # Future-proof against any change in case
1728 # and backwards compatibility with prior versions
1729 extractor = info_dict.get('extractor_key')
1730 if extractor is None:
1731 if 'id' in info_dict:
1732 extractor = info_dict.get('ie_key') # key in a playlist
1733 if extractor is None:
1734 return None # Incomplete video information
1735 return extractor.lower() + ' ' + info_dict['id']
1736
1737 def in_download_archive(self, info_dict):
1738 fn = self.params.get('download_archive')
1739 if fn is None:
1740 return False
1741
1742 vid_id = self._make_archive_id(info_dict)
1743 if vid_id is None:
1744 return False # Incomplete video information
1745
1746 try:
1747 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1748 for line in archive_file:
1749 if line.strip() == vid_id:
1750 return True
1751 except IOError as ioe:
1752 if ioe.errno != errno.ENOENT:
1753 raise
1754 return False
1755
1756 def record_download_archive(self, info_dict):
1757 fn = self.params.get('download_archive')
1758 if fn is None:
1759 return
1760 vid_id = self._make_archive_id(info_dict)
1761 assert vid_id
1762 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1763 archive_file.write(vid_id + '\n')
1764
1765 @staticmethod
1766 def format_resolution(format, default='unknown'):
1767 if format.get('vcodec') == 'none':
1768 return 'audio only'
1769 if format.get('resolution') is not None:
1770 return format['resolution']
1771 if format.get('height') is not None:
1772 if format.get('width') is not None:
1773 res = '%sx%s' % (format['width'], format['height'])
1774 else:
1775 res = '%sp' % format['height']
1776 elif format.get('width') is not None:
1777 res = '?x%d' % format['width']
1778 else:
1779 res = default
1780 return res
1781
1782 def _format_note(self, fdict):
1783 res = ''
1784 if fdict.get('ext') in ['f4f', 'f4m']:
1785 res += '(unsupported) '
1786 if fdict.get('format_note') is not None:
1787 res += fdict['format_note'] + ' '
1788 if fdict.get('tbr') is not None:
1789 res += '%4dk ' % fdict['tbr']
1790 if fdict.get('container') is not None:
1791 if res:
1792 res += ', '
1793 res += '%s container' % fdict['container']
1794 if (fdict.get('vcodec') is not None and
1795 fdict.get('vcodec') != 'none'):
1796 if res:
1797 res += ', '
1798 res += fdict['vcodec']
1799 if fdict.get('vbr') is not None:
1800 res += '@'
1801 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1802 res += 'video@'
1803 if fdict.get('vbr') is not None:
1804 res += '%4dk' % fdict['vbr']
1805 if fdict.get('fps') is not None:
1806 res += ', %sfps' % fdict['fps']
1807 if fdict.get('acodec') is not None:
1808 if res:
1809 res += ', '
1810 if fdict['acodec'] == 'none':
1811 res += 'video only'
1812 else:
1813 res += '%-5s' % fdict['acodec']
1814 elif fdict.get('abr') is not None:
1815 if res:
1816 res += ', '
1817 res += 'audio'
1818 if fdict.get('abr') is not None:
1819 res += '@%3dk' % fdict['abr']
1820 if fdict.get('asr') is not None:
1821 res += ' (%5dHz)' % fdict['asr']
1822 if fdict.get('filesize') is not None:
1823 if res:
1824 res += ', '
1825 res += format_bytes(fdict['filesize'])
1826 elif fdict.get('filesize_approx') is not None:
1827 if res:
1828 res += ', '
1829 res += '~' + format_bytes(fdict['filesize_approx'])
1830 return res
1831
1832 def list_formats(self, info_dict):
1833 formats = info_dict.get('formats', [info_dict])
1834 table = [
1835 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1836 for f in formats
1837 if f.get('preference') is None or f['preference'] >= -1000]
1838 if len(formats) > 1:
1839 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1840
1841 header_line = ['format code', 'extension', 'resolution', 'note']
1842 self.to_screen(
1843 '[info] Available formats for %s:\n%s' %
1844 (info_dict['id'], render_table(header_line, table)))
1845
1846 def list_thumbnails(self, info_dict):
1847 thumbnails = info_dict.get('thumbnails')
1848 if not thumbnails:
1849 tn_url = info_dict.get('thumbnail')
1850 if tn_url:
1851 thumbnails = [{'id': '0', 'url': tn_url}]
1852 else:
1853 self.to_screen(
1854 '[info] No thumbnails present for %s' % info_dict['id'])
1855 return
1856
1857 self.to_screen(
1858 '[info] Thumbnails for %s:' % info_dict['id'])
1859 self.to_screen(render_table(
1860 ['ID', 'width', 'height', 'URL'],
1861 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1862
1863 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1864 if not subtitles:
1865 self.to_screen('%s has no %s' % (video_id, name))
1866 return
1867 self.to_screen(
1868 'Available %s for %s:' % (name, video_id))
1869 self.to_screen(render_table(
1870 ['Language', 'formats'],
1871 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1872 for lang, formats in subtitles.items()]))
1873
1874 def urlopen(self, req):
1875 """ Start an HTTP download """
1876 if isinstance(req, compat_basestring):
1877 req = sanitized_Request(req)
1878 return self._opener.open(req, timeout=self._socket_timeout)
1879
1880 def print_debug_header(self):
1881 if not self.params.get('verbose'):
1882 return
1883
1884 if type('') is not compat_str:
1885 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1886 self.report_warning(
1887 'Your Python is broken! Update to a newer and supported version')
1888
1889 stdout_encoding = getattr(
1890 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1891 encoding_str = (
1892 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1893 locale.getpreferredencoding(),
1894 sys.getfilesystemencoding(),
1895 stdout_encoding,
1896 self.get_encoding()))
1897 write_string(encoding_str, encoding=None)
1898
1899 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1900 try:
1901 sp = subprocess.Popen(
1902 ['git', 'rev-parse', '--short', 'HEAD'],
1903 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1904 cwd=os.path.dirname(os.path.abspath(__file__)))
1905 out, err = sp.communicate()
1906 out = out.decode().strip()
1907 if re.match('[0-9a-f]+', out):
1908 self._write_string('[debug] Git HEAD: ' + out + '\n')
1909 except Exception:
1910 try:
1911 sys.exc_clear()
1912 except Exception:
1913 pass
1914 self._write_string('[debug] Python version %s - %s\n' % (
1915 platform.python_version(), platform_name()))
1916
1917 exe_versions = FFmpegPostProcessor.get_versions(self)
1918 exe_versions['rtmpdump'] = rtmpdump_version()
1919 exe_str = ', '.join(
1920 '%s %s' % (exe, v)
1921 for exe, v in sorted(exe_versions.items())
1922 if v
1923 )
1924 if not exe_str:
1925 exe_str = 'none'
1926 self._write_string('[debug] exe versions: %s\n' % exe_str)
1927
1928 proxy_map = {}
1929 for handler in self._opener.handlers:
1930 if hasattr(handler, 'proxies'):
1931 proxy_map.update(handler.proxies)
1932 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1933
1934 if self.params.get('call_home', False):
1935 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1936 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1937 latest_version = self.urlopen(
1938 'https://yt-dl.org/latest/version').read().decode('utf-8')
1939 if version_tuple(latest_version) > version_tuple(__version__):
1940 self.report_warning(
1941 'You are using an outdated version (newest version: %s)! '
1942 'See https://yt-dl.org/update if you need help updating.' %
1943 latest_version)
1944
1945 def _setup_opener(self):
1946 timeout_val = self.params.get('socket_timeout')
1947 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1948
1949 opts_cookiefile = self.params.get('cookiefile')
1950 opts_proxy = self.params.get('proxy')
1951
1952 if opts_cookiefile is None:
1953 self.cookiejar = compat_cookiejar.CookieJar()
1954 else:
1955 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1956 opts_cookiefile)
1957 if os.access(opts_cookiefile, os.R_OK):
1958 self.cookiejar.load()
1959
1960 cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
1961 if opts_proxy is not None:
1962 if opts_proxy == '':
1963 proxies = {}
1964 else:
1965 proxies = {'http': opts_proxy, 'https': opts_proxy}
1966 else:
1967 proxies = compat_urllib_request.getproxies()
1968 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1969 if 'http' in proxies and 'https' not in proxies:
1970 proxies['https'] = proxies['http']
1971 proxy_handler = PerRequestProxyHandler(proxies)
1972
1973 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1974 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1975 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1976 data_handler = compat_urllib_request_DataHandler()
1977 opener = compat_urllib_request.build_opener(
1978 proxy_handler, https_handler, cookie_processor, ydlh, data_handler)
1979
1980 # Delete the default user-agent header, which would otherwise apply in
1981 # cases where our custom HTTP handler doesn't come into play
1982 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1983 opener.addheaders = []
1984 self._opener = opener
1985
1986 def encode(self, s):
1987 if isinstance(s, bytes):
1988 return s # Already encoded
1989
1990 try:
1991 return s.encode(self.get_encoding())
1992 except UnicodeEncodeError as err:
1993 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1994 raise
1995
1996 def get_encoding(self):
1997 encoding = self.params.get('encoding')
1998 if encoding is None:
1999 encoding = preferredencoding()
2000 return encoding
2001
2002 def _write_thumbnails(self, info_dict, filename):
2003 if self.params.get('writethumbnail', False):
2004 thumbnails = info_dict.get('thumbnails')
2005 if thumbnails:
2006 thumbnails = [thumbnails[-1]]
2007 elif self.params.get('write_all_thumbnails', False):
2008 thumbnails = info_dict.get('thumbnails')
2009 else:
2010 return
2011
2012 if not thumbnails:
2013 # No thumbnails present, so return immediately
2014 return
2015
2016 for t in thumbnails:
2017 thumb_ext = determine_ext(t['url'], 'jpg')
2018 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2019 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2020 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2021
2022 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2023 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2024 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2025 else:
2026 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2027 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2028 try:
2029 uf = self.urlopen(t['url'])
2030 with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2031 shutil.copyfileobj(uf, thumbf)
2032 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2033 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2034 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2035 self.report_warning('Unable to download thumbnail "%s": %s' %
2036 (t['url'], compat_str(err)))