]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py
1783ce01bf683ab0cfa349a1b040362f88cc1a47
[youtubedl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import absolute_import, unicode_literals
5
6 import collections
7 import contextlib
8 import datetime
9 import errno
10 import fileinput
11 import io
12 import itertools
13 import json
14 import locale
15 import operator
16 import os
17 import platform
18 import re
19 import shutil
20 import subprocess
21 import socket
22 import sys
23 import time
24 import tokenize
25 import traceback
26
27 if os.name == 'nt':
28 import ctypes
29
30 from .compat import (
31 compat_cookiejar,
32 compat_expanduser,
33 compat_get_terminal_size,
34 compat_http_client,
35 compat_kwargs,
36 compat_str,
37 compat_tokenize_tokenize,
38 compat_urllib_error,
39 compat_urllib_request,
40 compat_urllib_request_DataHandler,
41 )
42 from .utils import (
43 ContentTooShortError,
44 date_from_str,
45 DateRange,
46 DEFAULT_OUTTMPL,
47 determine_ext,
48 DownloadError,
49 encodeFilename,
50 ExtractorError,
51 format_bytes,
52 formatSeconds,
53 locked_file,
54 make_HTTPS_handler,
55 MaxDownloadsReached,
56 PagedList,
57 parse_filesize,
58 PerRequestProxyHandler,
59 PostProcessingError,
60 platform_name,
61 preferredencoding,
62 render_table,
63 SameFileError,
64 sanitize_filename,
65 sanitize_path,
66 std_headers,
67 subtitles_filename,
68 UnavailableVideoError,
69 url_basename,
70 version_tuple,
71 write_json_file,
72 write_string,
73 YoutubeDLCookieProcessor,
74 YoutubeDLHandler,
75 prepend_extension,
76 replace_extension,
77 args_to_str,
78 age_restricted,
79 )
80 from .cache import Cache
81 from .extractor import get_info_extractor, gen_extractors
82 from .downloader import get_suitable_downloader
83 from .downloader.rtmp import rtmpdump_version
84 from .postprocessor import (
85 FFmpegFixupM4aPP,
86 FFmpegFixupStretchedPP,
87 FFmpegMergerPP,
88 FFmpegPostProcessor,
89 get_postprocessor,
90 )
91 from .version import __version__
92
93
94 class YoutubeDL(object):
95 """YoutubeDL class.
96
97 YoutubeDL objects are the ones responsible of downloading the
98 actual video file and writing it to disk if the user has requested
99 it, among some other tasks. In most cases there should be one per
100 program. As, given a video URL, the downloader doesn't know how to
101 extract all the needed information, task that InfoExtractors do, it
102 has to pass the URL to one of them.
103
104 For this, YoutubeDL objects have a method that allows
105 InfoExtractors to be registered in a given order. When it is passed
106 a URL, the YoutubeDL object handles it to the first InfoExtractor it
107 finds that reports being able to handle it. The InfoExtractor extracts
108 all the information about the video or videos the URL refers to, and
109 YoutubeDL process the extracted information, possibly using a File
110 Downloader to download the video.
111
112 YoutubeDL objects accept a lot of parameters. In order not to saturate
113 the object constructor with arguments, it receives a dictionary of
114 options instead. These options are available through the params
115 attribute for the InfoExtractors to use. The YoutubeDL also
116 registers itself as the downloader in charge for the InfoExtractors
117 that are added to it, so this is a "mutual registration".
118
119 Available options:
120
121 username: Username for authentication purposes.
122 password: Password for authentication purposes.
123 videopassword: Password for accessing a video.
124 usenetrc: Use netrc for authentication instead.
125 verbose: Print additional info to stdout.
126 quiet: Do not print messages to stdout.
127 no_warnings: Do not print out anything for warnings.
128 forceurl: Force printing final URL.
129 forcetitle: Force printing title.
130 forceid: Force printing ID.
131 forcethumbnail: Force printing thumbnail URL.
132 forcedescription: Force printing description.
133 forcefilename: Force printing final filename.
134 forceduration: Force printing duration.
135 forcejson: Force printing info_dict as JSON.
136 dump_single_json: Force printing the info_dict of the whole playlist
137 (or video) as a single JSON line.
138 simulate: Do not download the video files.
139 format: Video format code. See options.py for more information.
140 outtmpl: Template for output names.
141 restrictfilenames: Do not allow "&" and spaces in file names
142 ignoreerrors: Do not stop on download errors.
143 force_generic_extractor: Force downloader to use the generic extractor
144 nooverwrites: Prevent overwriting files.
145 playliststart: Playlist item to start at.
146 playlistend: Playlist item to end at.
147 playlist_items: Specific indices of playlist to download.
148 playlistreverse: Download playlist items in reverse order.
149 matchtitle: Download only matching titles.
150 rejecttitle: Reject downloads for matching titles.
151 logger: Log messages to a logging.Logger instance.
152 logtostderr: Log messages to stderr instead of stdout.
153 writedescription: Write the video description to a .description file
154 writeinfojson: Write the video description to a .info.json file
155 writeannotations: Write the video annotations to a .annotations.xml file
156 writethumbnail: Write the thumbnail image to a file
157 write_all_thumbnails: Write all thumbnail formats to files
158 writesubtitles: Write the video subtitles to a file
159 writeautomaticsub: Write the automatic subtitles to a file
160 allsubtitles: Downloads all the subtitles of the video
161 (requires writesubtitles or writeautomaticsub)
162 listsubtitles: Lists all available subtitles for the video
163 subtitlesformat: The format code for subtitles
164 subtitleslangs: List of languages of the subtitles to download
165 keepvideo: Keep the video file after post-processing
166 daterange: A DateRange object, download only if the upload_date is in the range.
167 skip_download: Skip the actual download of the video file
168 cachedir: Location of the cache files in the filesystem.
169 False to disable filesystem cache.
170 noplaylist: Download single video instead of a playlist if in doubt.
171 age_limit: An integer representing the user's age in years.
172 Unsuitable videos for the given age are skipped.
173 min_views: An integer representing the minimum view count the video
174 must have in order to not be skipped.
175 Videos without view count information are always
176 downloaded. None for no limit.
177 max_views: An integer representing the maximum view count.
178 Videos that are more popular than that are not
179 downloaded.
180 Videos without view count information are always
181 downloaded. None for no limit.
182 download_archive: File name of a file where all downloads are recorded.
183 Videos already present in the file are not downloaded
184 again.
185 cookiefile: File name where cookies should be read from and dumped to.
186 nocheckcertificate:Do not verify SSL certificates
187 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
188 At the moment, this is only supported by YouTube.
189 proxy: URL of the proxy server to use
190 cn_verification_proxy: URL of the proxy to use for IP address verification
191 on Chinese sites. (Experimental)
192 socket_timeout: Time to wait for unresponsive hosts, in seconds
193 bidi_workaround: Work around buggy terminals without bidirectional text
194 support, using fridibi
195 debug_printtraffic:Print out sent and received HTTP traffic
196 include_ads: Download ads as well
197 default_search: Prepend this string if an input url is not valid.
198 'auto' for elaborate guessing
199 encoding: Use this encoding instead of the system-specified.
200 extract_flat: Do not resolve URLs, return the immediate result.
201 Pass in 'in_playlist' to only show this behavior for
202 playlist items.
203 postprocessors: A list of dictionaries, each with an entry
204 * key: The name of the postprocessor. See
205 youtube_dl/postprocessor/__init__.py for a list.
206 as well as any further keyword arguments for the
207 postprocessor.
208 progress_hooks: A list of functions that get called on download
209 progress, with a dictionary with the entries
210 * status: One of "downloading", "error", or "finished".
211 Check this first and ignore unknown values.
212
213 If status is one of "downloading", or "finished", the
214 following properties may also be present:
215 * filename: The final filename (always present)
216 * tmpfilename: The filename we're currently writing to
217 * downloaded_bytes: Bytes on disk
218 * total_bytes: Size of the whole file, None if unknown
219 * total_bytes_estimate: Guess of the eventual file size,
220 None if unavailable.
221 * elapsed: The number of seconds since download started.
222 * eta: The estimated time in seconds, None if unknown
223 * speed: The download speed in bytes/second, None if
224 unknown
225 * fragment_index: The counter of the currently
226 downloaded video fragment.
227 * fragment_count: The number of fragments (= individual
228 files that will be merged)
229
230 Progress hooks are guaranteed to be called at least once
231 (with status "finished") if the download is successful.
232 merge_output_format: Extension to use when merging formats.
233 fixup: Automatically correct known faults of the file.
234 One of:
235 - "never": do nothing
236 - "warn": only emit a warning
237 - "detect_or_warn": check whether we can do anything
238 about it, warn otherwise (default)
239 source_address: (Experimental) Client-side IP address to bind to.
240 call_home: Boolean, true iff we are allowed to contact the
241 youtube-dl servers for debugging.
242 sleep_interval: Number of seconds to sleep before each download.
243 listformats: Print an overview of available video formats and exit.
244 list_thumbnails: Print a table of all thumbnails and exit.
245 match_filter: A function that gets called with the info_dict of
246 every video.
247 If it returns a message, the video is ignored.
248 If it returns None, the video is downloaded.
249 match_filter_func in utils.py is one example for this.
250 no_color: Do not emit color codes in output.
251
252 The following options determine which downloader is picked:
253 external_downloader: Executable of the external downloader to call.
254 None or unset for standard (built-in) downloader.
255 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
256
257 The following parameters are not used by YoutubeDL itself, they are used by
258 the downloader (see youtube_dl/downloader/common.py):
259 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
260 noresizebuffer, retries, continuedl, noprogress, consoletitle,
261 xattr_set_filesize, external_downloader_args.
262
263 The following options are used by the post processors:
264 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
265 otherwise prefer avconv.
266 postprocessor_args: A list of additional command-line arguments for the
267 postprocessor.
268 """
269
270 params = None
271 _ies = []
272 _pps = []
273 _download_retcode = None
274 _num_downloads = None
275 _screen_file = None
276
277 def __init__(self, params=None, auto_init=True):
278 """Create a FileDownloader object with the given options."""
279 if params is None:
280 params = {}
281 self._ies = []
282 self._ies_instances = {}
283 self._pps = []
284 self._progress_hooks = []
285 self._download_retcode = 0
286 self._num_downloads = 0
287 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
288 self._err_file = sys.stderr
289 self.params = {
290 # Default parameters
291 'nocheckcertificate': False,
292 }
293 self.params.update(params)
294 self.cache = Cache(self)
295
296 if params.get('bidi_workaround', False):
297 try:
298 import pty
299 master, slave = pty.openpty()
300 width = compat_get_terminal_size().columns
301 if width is None:
302 width_args = []
303 else:
304 width_args = ['-w', str(width)]
305 sp_kwargs = dict(
306 stdin=subprocess.PIPE,
307 stdout=slave,
308 stderr=self._err_file)
309 try:
310 self._output_process = subprocess.Popen(
311 ['bidiv'] + width_args, **sp_kwargs
312 )
313 except OSError:
314 self._output_process = subprocess.Popen(
315 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
316 self._output_channel = os.fdopen(master, 'rb')
317 except OSError as ose:
318 if ose.errno == 2:
319 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
320 else:
321 raise
322
323 if (sys.version_info >= (3,) and sys.platform != 'win32' and
324 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
325 not params.get('restrictfilenames', False)):
326 # On Python 3, the Unicode filesystem API will throw errors (#1474)
327 self.report_warning(
328 'Assuming --restrict-filenames since file system encoding '
329 'cannot encode all characters. '
330 'Set the LC_ALL environment variable to fix this.')
331 self.params['restrictfilenames'] = True
332
333 if isinstance(params.get('outtmpl'), bytes):
334 self.report_warning(
335 'Parameter outtmpl is bytes, but should be a unicode string. '
336 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
337
338 self._setup_opener()
339
340 if auto_init:
341 self.print_debug_header()
342 self.add_default_info_extractors()
343
344 for pp_def_raw in self.params.get('postprocessors', []):
345 pp_class = get_postprocessor(pp_def_raw['key'])
346 pp_def = dict(pp_def_raw)
347 del pp_def['key']
348 pp = pp_class(self, **compat_kwargs(pp_def))
349 self.add_post_processor(pp)
350
351 for ph in self.params.get('progress_hooks', []):
352 self.add_progress_hook(ph)
353
354 def warn_if_short_id(self, argv):
355 # short YouTube ID starting with dash?
356 idxs = [
357 i for i, a in enumerate(argv)
358 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
359 if idxs:
360 correct_argv = (
361 ['youtube-dl'] +
362 [a for i, a in enumerate(argv) if i not in idxs] +
363 ['--'] + [argv[i] for i in idxs]
364 )
365 self.report_warning(
366 'Long argument string detected. '
367 'Use -- to separate parameters and URLs, like this:\n%s\n' %
368 args_to_str(correct_argv))
369
370 def add_info_extractor(self, ie):
371 """Add an InfoExtractor object to the end of the list."""
372 self._ies.append(ie)
373 self._ies_instances[ie.ie_key()] = ie
374 ie.set_downloader(self)
375
376 def get_info_extractor(self, ie_key):
377 """
378 Get an instance of an IE with name ie_key, it will try to get one from
379 the _ies list, if there's no instance it will create a new one and add
380 it to the extractor list.
381 """
382 ie = self._ies_instances.get(ie_key)
383 if ie is None:
384 ie = get_info_extractor(ie_key)()
385 self.add_info_extractor(ie)
386 return ie
387
388 def add_default_info_extractors(self):
389 """
390 Add the InfoExtractors returned by gen_extractors to the end of the list
391 """
392 for ie in gen_extractors():
393 self.add_info_extractor(ie)
394
395 def add_post_processor(self, pp):
396 """Add a PostProcessor object to the end of the chain."""
397 self._pps.append(pp)
398 pp.set_downloader(self)
399
400 def add_progress_hook(self, ph):
401 """Add the progress hook (currently only for the file downloader)"""
402 self._progress_hooks.append(ph)
403
404 def _bidi_workaround(self, message):
405 if not hasattr(self, '_output_channel'):
406 return message
407
408 assert hasattr(self, '_output_process')
409 assert isinstance(message, compat_str)
410 line_count = message.count('\n') + 1
411 self._output_process.stdin.write((message + '\n').encode('utf-8'))
412 self._output_process.stdin.flush()
413 res = ''.join(self._output_channel.readline().decode('utf-8')
414 for _ in range(line_count))
415 return res[:-len('\n')]
416
417 def to_screen(self, message, skip_eol=False):
418 """Print message to stdout if not in quiet mode."""
419 return self.to_stdout(message, skip_eol, check_quiet=True)
420
421 def _write_string(self, s, out=None):
422 write_string(s, out=out, encoding=self.params.get('encoding'))
423
424 def to_stdout(self, message, skip_eol=False, check_quiet=False):
425 """Print message to stdout if not in quiet mode."""
426 if self.params.get('logger'):
427 self.params['logger'].debug(message)
428 elif not check_quiet or not self.params.get('quiet', False):
429 message = self._bidi_workaround(message)
430 terminator = ['\n', ''][skip_eol]
431 output = message + terminator
432
433 self._write_string(output, self._screen_file)
434
435 def to_stderr(self, message):
436 """Print message to stderr."""
437 assert isinstance(message, compat_str)
438 if self.params.get('logger'):
439 self.params['logger'].error(message)
440 else:
441 message = self._bidi_workaround(message)
442 output = message + '\n'
443 self._write_string(output, self._err_file)
444
445 def to_console_title(self, message):
446 if not self.params.get('consoletitle', False):
447 return
448 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
449 # c_wchar_p() might not be necessary if `message` is
450 # already of type unicode()
451 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
452 elif 'TERM' in os.environ:
453 self._write_string('\033]0;%s\007' % message, self._screen_file)
454
455 def save_console_title(self):
456 if not self.params.get('consoletitle', False):
457 return
458 if 'TERM' in os.environ:
459 # Save the title on stack
460 self._write_string('\033[22;0t', self._screen_file)
461
462 def restore_console_title(self):
463 if not self.params.get('consoletitle', False):
464 return
465 if 'TERM' in os.environ:
466 # Restore the title from stack
467 self._write_string('\033[23;0t', self._screen_file)
468
469 def __enter__(self):
470 self.save_console_title()
471 return self
472
473 def __exit__(self, *args):
474 self.restore_console_title()
475
476 if self.params.get('cookiefile') is not None:
477 self.cookiejar.save()
478
479 def trouble(self, message=None, tb=None):
480 """Determine action to take when a download problem appears.
481
482 Depending on if the downloader has been configured to ignore
483 download errors or not, this method may throw an exception or
484 not when errors are found, after printing the message.
485
486 tb, if given, is additional traceback information.
487 """
488 if message is not None:
489 self.to_stderr(message)
490 if self.params.get('verbose'):
491 if tb is None:
492 if sys.exc_info()[0]: # if .trouble has been called from an except block
493 tb = ''
494 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
495 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
496 tb += compat_str(traceback.format_exc())
497 else:
498 tb_data = traceback.format_list(traceback.extract_stack())
499 tb = ''.join(tb_data)
500 self.to_stderr(tb)
501 if not self.params.get('ignoreerrors', False):
502 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
503 exc_info = sys.exc_info()[1].exc_info
504 else:
505 exc_info = sys.exc_info()
506 raise DownloadError(message, exc_info)
507 self._download_retcode = 1
508
509 def report_warning(self, message):
510 '''
511 Print the message to stderr, it will be prefixed with 'WARNING:'
512 If stderr is a tty file the 'WARNING:' will be colored
513 '''
514 if self.params.get('logger') is not None:
515 self.params['logger'].warning(message)
516 else:
517 if self.params.get('no_warnings'):
518 return
519 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
520 _msg_header = '\033[0;33mWARNING:\033[0m'
521 else:
522 _msg_header = 'WARNING:'
523 warning_message = '%s %s' % (_msg_header, message)
524 self.to_stderr(warning_message)
525
526 def report_error(self, message, tb=None):
527 '''
528 Do the same as trouble, but prefixes the message with 'ERROR:', colored
529 in red if stderr is a tty file.
530 '''
531 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
532 _msg_header = '\033[0;31mERROR:\033[0m'
533 else:
534 _msg_header = 'ERROR:'
535 error_message = '%s %s' % (_msg_header, message)
536 self.trouble(error_message, tb)
537
538 def report_file_already_downloaded(self, file_name):
539 """Report file has already been fully downloaded."""
540 try:
541 self.to_screen('[download] %s has already been downloaded' % file_name)
542 except UnicodeEncodeError:
543 self.to_screen('[download] The file has already been downloaded')
544
545 def prepare_filename(self, info_dict):
546 """Generate the output filename."""
547 try:
548 template_dict = dict(info_dict)
549
550 template_dict['epoch'] = int(time.time())
551 autonumber_size = self.params.get('autonumber_size')
552 if autonumber_size is None:
553 autonumber_size = 5
554 autonumber_templ = '%0' + str(autonumber_size) + 'd'
555 template_dict['autonumber'] = autonumber_templ % self._num_downloads
556 if template_dict.get('playlist_index') is not None:
557 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
558 if template_dict.get('resolution') is None:
559 if template_dict.get('width') and template_dict.get('height'):
560 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
561 elif template_dict.get('height'):
562 template_dict['resolution'] = '%sp' % template_dict['height']
563 elif template_dict.get('width'):
564 template_dict['resolution'] = '?x%d' % template_dict['width']
565
566 sanitize = lambda k, v: sanitize_filename(
567 compat_str(v),
568 restricted=self.params.get('restrictfilenames'),
569 is_id=(k == 'id'))
570 template_dict = dict((k, sanitize(k, v))
571 for k, v in template_dict.items()
572 if v is not None)
573 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
574
575 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
576 tmpl = compat_expanduser(outtmpl)
577 filename = tmpl % template_dict
578 # Temporary fix for #4787
579 # 'Treat' all problem characters by passing filename through preferredencoding
580 # to workaround encoding issues with subprocess on python2 @ Windows
581 if sys.version_info < (3, 0) and sys.platform == 'win32':
582 filename = encodeFilename(filename, True).decode(preferredencoding())
583 return sanitize_path(filename)
584 except ValueError as err:
585 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
586 return None
587
588 def _match_entry(self, info_dict, incomplete):
589 """ Returns None iff the file should be downloaded """
590
591 video_title = info_dict.get('title', info_dict.get('id', 'video'))
592 if 'title' in info_dict:
593 # This can happen when we're just evaluating the playlist
594 title = info_dict['title']
595 matchtitle = self.params.get('matchtitle', False)
596 if matchtitle:
597 if not re.search(matchtitle, title, re.IGNORECASE):
598 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
599 rejecttitle = self.params.get('rejecttitle', False)
600 if rejecttitle:
601 if re.search(rejecttitle, title, re.IGNORECASE):
602 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
603 date = info_dict.get('upload_date', None)
604 if date is not None:
605 dateRange = self.params.get('daterange', DateRange())
606 if date not in dateRange:
607 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
608 view_count = info_dict.get('view_count', None)
609 if view_count is not None:
610 min_views = self.params.get('min_views')
611 if min_views is not None and view_count < min_views:
612 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
613 max_views = self.params.get('max_views')
614 if max_views is not None and view_count > max_views:
615 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
616 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
617 return 'Skipping "%s" because it is age restricted' % video_title
618 if self.in_download_archive(info_dict):
619 return '%s has already been recorded in archive' % video_title
620
621 if not incomplete:
622 match_filter = self.params.get('match_filter')
623 if match_filter is not None:
624 ret = match_filter(info_dict)
625 if ret is not None:
626 return ret
627
628 return None
629
630 @staticmethod
631 def add_extra_info(info_dict, extra_info):
632 '''Set the keys from extra_info in info dict if they are missing'''
633 for key, value in extra_info.items():
634 info_dict.setdefault(key, value)
635
636 def extract_info(self, url, download=True, ie_key=None, extra_info={},
637 process=True, force_generic_extractor=False):
638 '''
639 Returns a list with a dictionary for each video we find.
640 If 'download', also downloads the videos.
641 extra_info is a dict containing the extra values to add to each result
642 '''
643
644 if not ie_key and force_generic_extractor:
645 ie_key = 'Generic'
646
647 if ie_key:
648 ies = [self.get_info_extractor(ie_key)]
649 else:
650 ies = self._ies
651
652 for ie in ies:
653 if not ie.suitable(url):
654 continue
655
656 if not ie.working():
657 self.report_warning('The program functionality for this site has been marked as broken, '
658 'and will probably not work.')
659
660 try:
661 ie_result = ie.extract(url)
662 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
663 break
664 if isinstance(ie_result, list):
665 # Backwards compatibility: old IE result format
666 ie_result = {
667 '_type': 'compat_list',
668 'entries': ie_result,
669 }
670 self.add_default_extra_info(ie_result, ie, url)
671 if process:
672 return self.process_ie_result(ie_result, download, extra_info)
673 else:
674 return ie_result
675 except ExtractorError as de: # An error we somewhat expected
676 self.report_error(compat_str(de), de.format_traceback())
677 break
678 except MaxDownloadsReached:
679 raise
680 except Exception as e:
681 if self.params.get('ignoreerrors', False):
682 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
683 break
684 else:
685 raise
686 else:
687 self.report_error('no suitable InfoExtractor for URL %s' % url)
688
689 def add_default_extra_info(self, ie_result, ie, url):
690 self.add_extra_info(ie_result, {
691 'extractor': ie.IE_NAME,
692 'webpage_url': url,
693 'webpage_url_basename': url_basename(url),
694 'extractor_key': ie.ie_key(),
695 })
696
697 def process_ie_result(self, ie_result, download=True, extra_info={}):
698 """
699 Take the result of the ie(may be modified) and resolve all unresolved
700 references (URLs, playlist items).
701
702 It will also download the videos if 'download'.
703 Returns the resolved ie_result.
704 """
705
706 result_type = ie_result.get('_type', 'video')
707
708 if result_type in ('url', 'url_transparent'):
709 extract_flat = self.params.get('extract_flat', False)
710 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
711 extract_flat is True):
712 if self.params.get('forcejson', False):
713 self.to_stdout(json.dumps(ie_result))
714 return ie_result
715
716 if result_type == 'video':
717 self.add_extra_info(ie_result, extra_info)
718 return self.process_video_result(ie_result, download=download)
719 elif result_type == 'url':
720 # We have to add extra_info to the results because it may be
721 # contained in a playlist
722 return self.extract_info(ie_result['url'],
723 download,
724 ie_key=ie_result.get('ie_key'),
725 extra_info=extra_info)
726 elif result_type == 'url_transparent':
727 # Use the information from the embedding page
728 info = self.extract_info(
729 ie_result['url'], ie_key=ie_result.get('ie_key'),
730 extra_info=extra_info, download=False, process=False)
731
732 force_properties = dict(
733 (k, v) for k, v in ie_result.items() if v is not None)
734 for f in ('_type', 'url'):
735 if f in force_properties:
736 del force_properties[f]
737 new_result = info.copy()
738 new_result.update(force_properties)
739
740 assert new_result.get('_type') != 'url_transparent'
741
742 return self.process_ie_result(
743 new_result, download=download, extra_info=extra_info)
744 elif result_type == 'playlist' or result_type == 'multi_video':
745 # We process each entry in the playlist
746 playlist = ie_result.get('title', None) or ie_result.get('id', None)
747 self.to_screen('[download] Downloading playlist: %s' % playlist)
748
749 playlist_results = []
750
751 playliststart = self.params.get('playliststart', 1) - 1
752 playlistend = self.params.get('playlistend', None)
753 # For backwards compatibility, interpret -1 as whole list
754 if playlistend == -1:
755 playlistend = None
756
757 playlistitems_str = self.params.get('playlist_items', None)
758 playlistitems = None
759 if playlistitems_str is not None:
760 def iter_playlistitems(format):
761 for string_segment in format.split(','):
762 if '-' in string_segment:
763 start, end = string_segment.split('-')
764 for item in range(int(start), int(end) + 1):
765 yield int(item)
766 else:
767 yield int(string_segment)
768 playlistitems = iter_playlistitems(playlistitems_str)
769
770 ie_entries = ie_result['entries']
771 if isinstance(ie_entries, list):
772 n_all_entries = len(ie_entries)
773 if playlistitems:
774 entries = [
775 ie_entries[i - 1] for i in playlistitems
776 if -n_all_entries <= i - 1 < n_all_entries]
777 else:
778 entries = ie_entries[playliststart:playlistend]
779 n_entries = len(entries)
780 self.to_screen(
781 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
782 (ie_result['extractor'], playlist, n_all_entries, n_entries))
783 elif isinstance(ie_entries, PagedList):
784 if playlistitems:
785 entries = []
786 for item in playlistitems:
787 entries.extend(ie_entries.getslice(
788 item - 1, item
789 ))
790 else:
791 entries = ie_entries.getslice(
792 playliststart, playlistend)
793 n_entries = len(entries)
794 self.to_screen(
795 "[%s] playlist %s: Downloading %d videos" %
796 (ie_result['extractor'], playlist, n_entries))
797 else: # iterable
798 if playlistitems:
799 entry_list = list(ie_entries)
800 entries = [entry_list[i - 1] for i in playlistitems]
801 else:
802 entries = list(itertools.islice(
803 ie_entries, playliststart, playlistend))
804 n_entries = len(entries)
805 self.to_screen(
806 "[%s] playlist %s: Downloading %d videos" %
807 (ie_result['extractor'], playlist, n_entries))
808
809 if self.params.get('playlistreverse', False):
810 entries = entries[::-1]
811
812 for i, entry in enumerate(entries, 1):
813 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
814 extra = {
815 'n_entries': n_entries,
816 'playlist': playlist,
817 'playlist_id': ie_result.get('id'),
818 'playlist_title': ie_result.get('title'),
819 'playlist_index': i + playliststart,
820 'extractor': ie_result['extractor'],
821 'webpage_url': ie_result['webpage_url'],
822 'webpage_url_basename': url_basename(ie_result['webpage_url']),
823 'extractor_key': ie_result['extractor_key'],
824 }
825
826 reason = self._match_entry(entry, incomplete=True)
827 if reason is not None:
828 self.to_screen('[download] ' + reason)
829 continue
830
831 entry_result = self.process_ie_result(entry,
832 download=download,
833 extra_info=extra)
834 playlist_results.append(entry_result)
835 ie_result['entries'] = playlist_results
836 return ie_result
837 elif result_type == 'compat_list':
838 self.report_warning(
839 'Extractor %s returned a compat_list result. '
840 'It needs to be updated.' % ie_result.get('extractor'))
841
842 def _fixup(r):
843 self.add_extra_info(
844 r,
845 {
846 'extractor': ie_result['extractor'],
847 'webpage_url': ie_result['webpage_url'],
848 'webpage_url_basename': url_basename(ie_result['webpage_url']),
849 'extractor_key': ie_result['extractor_key'],
850 }
851 )
852 return r
853 ie_result['entries'] = [
854 self.process_ie_result(_fixup(r), download, extra_info)
855 for r in ie_result['entries']
856 ]
857 return ie_result
858 else:
859 raise Exception('Invalid result type: %s' % result_type)
860
861 def _build_format_filter(self, filter_spec):
862 " Returns a function to filter the formats according to the filter_spec "
863
864 OPERATORS = {
865 '<': operator.lt,
866 '<=': operator.le,
867 '>': operator.gt,
868 '>=': operator.ge,
869 '=': operator.eq,
870 '!=': operator.ne,
871 }
872 operator_rex = re.compile(r'''(?x)\s*
873 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
874 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
875 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
876 $
877 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
878 m = operator_rex.search(filter_spec)
879 if m:
880 try:
881 comparison_value = int(m.group('value'))
882 except ValueError:
883 comparison_value = parse_filesize(m.group('value'))
884 if comparison_value is None:
885 comparison_value = parse_filesize(m.group('value') + 'B')
886 if comparison_value is None:
887 raise ValueError(
888 'Invalid value %r in format specification %r' % (
889 m.group('value'), filter_spec))
890 op = OPERATORS[m.group('op')]
891
892 if not m:
893 STR_OPERATORS = {
894 '=': operator.eq,
895 '!=': operator.ne,
896 }
897 str_operator_rex = re.compile(r'''(?x)
898 \s*(?P<key>ext|acodec|vcodec|container|protocol)
899 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
900 \s*(?P<value>[a-zA-Z0-9_-]+)
901 \s*$
902 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
903 m = str_operator_rex.search(filter_spec)
904 if m:
905 comparison_value = m.group('value')
906 op = STR_OPERATORS[m.group('op')]
907
908 if not m:
909 raise ValueError('Invalid filter specification %r' % filter_spec)
910
911 def _filter(f):
912 actual_value = f.get(m.group('key'))
913 if actual_value is None:
914 return m.group('none_inclusive')
915 return op(actual_value, comparison_value)
916 return _filter
917
918 def build_format_selector(self, format_spec):
919 def syntax_error(note, start):
920 message = (
921 'Invalid format specification: '
922 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
923 return SyntaxError(message)
924
925 PICKFIRST = 'PICKFIRST'
926 MERGE = 'MERGE'
927 SINGLE = 'SINGLE'
928 GROUP = 'GROUP'
929 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
930
931 def _parse_filter(tokens):
932 filter_parts = []
933 for type, string, start, _, _ in tokens:
934 if type == tokenize.OP and string == ']':
935 return ''.join(filter_parts)
936 else:
937 filter_parts.append(string)
938
939 def _remove_unused_ops(tokens):
940 # Remove operators that we don't use and join them with the sourrounding strings
941 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
942 ALLOWED_OPS = ('/', '+', ',', '(', ')')
943 last_string, last_start, last_end, last_line = None, None, None, None
944 for type, string, start, end, line in tokens:
945 if type == tokenize.OP and string == '[':
946 if last_string:
947 yield tokenize.NAME, last_string, last_start, last_end, last_line
948 last_string = None
949 yield type, string, start, end, line
950 # everything inside brackets will be handled by _parse_filter
951 for type, string, start, end, line in tokens:
952 yield type, string, start, end, line
953 if type == tokenize.OP and string == ']':
954 break
955 elif type == tokenize.OP and string in ALLOWED_OPS:
956 if last_string:
957 yield tokenize.NAME, last_string, last_start, last_end, last_line
958 last_string = None
959 yield type, string, start, end, line
960 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
961 if not last_string:
962 last_string = string
963 last_start = start
964 last_end = end
965 else:
966 last_string += string
967 if last_string:
968 yield tokenize.NAME, last_string, last_start, last_end, last_line
969
970 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
971 selectors = []
972 current_selector = None
973 for type, string, start, _, _ in tokens:
974 # ENCODING is only defined in python 3.x
975 if type == getattr(tokenize, 'ENCODING', None):
976 continue
977 elif type in [tokenize.NAME, tokenize.NUMBER]:
978 current_selector = FormatSelector(SINGLE, string, [])
979 elif type == tokenize.OP:
980 if string == ')':
981 if not inside_group:
982 # ')' will be handled by the parentheses group
983 tokens.restore_last_token()
984 break
985 elif inside_merge and string in ['/', ',']:
986 tokens.restore_last_token()
987 break
988 elif inside_choice and string == ',':
989 tokens.restore_last_token()
990 break
991 elif string == ',':
992 if not current_selector:
993 raise syntax_error('"," must follow a format selector', start)
994 selectors.append(current_selector)
995 current_selector = None
996 elif string == '/':
997 if not current_selector:
998 raise syntax_error('"/" must follow a format selector', start)
999 first_choice = current_selector
1000 second_choice = _parse_format_selection(tokens, inside_choice=True)
1001 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1002 elif string == '[':
1003 if not current_selector:
1004 current_selector = FormatSelector(SINGLE, 'best', [])
1005 format_filter = _parse_filter(tokens)
1006 current_selector.filters.append(format_filter)
1007 elif string == '(':
1008 if current_selector:
1009 raise syntax_error('Unexpected "("', start)
1010 group = _parse_format_selection(tokens, inside_group=True)
1011 current_selector = FormatSelector(GROUP, group, [])
1012 elif string == '+':
1013 video_selector = current_selector
1014 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1015 if not video_selector or not audio_selector:
1016 raise syntax_error('"+" must be between two format selectors', start)
1017 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1018 else:
1019 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1020 elif type == tokenize.ENDMARKER:
1021 break
1022 if current_selector:
1023 selectors.append(current_selector)
1024 return selectors
1025
1026 def _build_selector_function(selector):
1027 if isinstance(selector, list):
1028 fs = [_build_selector_function(s) for s in selector]
1029
1030 def selector_function(formats):
1031 for f in fs:
1032 for format in f(formats):
1033 yield format
1034 return selector_function
1035 elif selector.type == GROUP:
1036 selector_function = _build_selector_function(selector.selector)
1037 elif selector.type == PICKFIRST:
1038 fs = [_build_selector_function(s) for s in selector.selector]
1039
1040 def selector_function(formats):
1041 for f in fs:
1042 picked_formats = list(f(formats))
1043 if picked_formats:
1044 return picked_formats
1045 return []
1046 elif selector.type == SINGLE:
1047 format_spec = selector.selector
1048
1049 def selector_function(formats):
1050 formats = list(formats)
1051 if not formats:
1052 return
1053 if format_spec == 'all':
1054 for f in formats:
1055 yield f
1056 elif format_spec in ['best', 'worst', None]:
1057 format_idx = 0 if format_spec == 'worst' else -1
1058 audiovideo_formats = [
1059 f for f in formats
1060 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1061 if audiovideo_formats:
1062 yield audiovideo_formats[format_idx]
1063 # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1064 elif (all(f.get('acodec') != 'none' for f in formats) or
1065 all(f.get('vcodec') != 'none' for f in formats)):
1066 yield formats[format_idx]
1067 elif format_spec == 'bestaudio':
1068 audio_formats = [
1069 f for f in formats
1070 if f.get('vcodec') == 'none']
1071 if audio_formats:
1072 yield audio_formats[-1]
1073 elif format_spec == 'worstaudio':
1074 audio_formats = [
1075 f for f in formats
1076 if f.get('vcodec') == 'none']
1077 if audio_formats:
1078 yield audio_formats[0]
1079 elif format_spec == 'bestvideo':
1080 video_formats = [
1081 f for f in formats
1082 if f.get('acodec') == 'none']
1083 if video_formats:
1084 yield video_formats[-1]
1085 elif format_spec == 'worstvideo':
1086 video_formats = [
1087 f for f in formats
1088 if f.get('acodec') == 'none']
1089 if video_formats:
1090 yield video_formats[0]
1091 else:
1092 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1093 if format_spec in extensions:
1094 filter_f = lambda f: f['ext'] == format_spec
1095 else:
1096 filter_f = lambda f: f['format_id'] == format_spec
1097 matches = list(filter(filter_f, formats))
1098 if matches:
1099 yield matches[-1]
1100 elif selector.type == MERGE:
1101 def _merge(formats_info):
1102 format_1, format_2 = [f['format_id'] for f in formats_info]
1103 # The first format must contain the video and the
1104 # second the audio
1105 if formats_info[0].get('vcodec') == 'none':
1106 self.report_error('The first format must '
1107 'contain the video, try using '
1108 '"-f %s+%s"' % (format_2, format_1))
1109 return
1110 output_ext = (
1111 formats_info[0]['ext']
1112 if self.params.get('merge_output_format') is None
1113 else self.params['merge_output_format'])
1114 return {
1115 'requested_formats': formats_info,
1116 'format': '%s+%s' % (formats_info[0].get('format'),
1117 formats_info[1].get('format')),
1118 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1119 formats_info[1].get('format_id')),
1120 'width': formats_info[0].get('width'),
1121 'height': formats_info[0].get('height'),
1122 'resolution': formats_info[0].get('resolution'),
1123 'fps': formats_info[0].get('fps'),
1124 'vcodec': formats_info[0].get('vcodec'),
1125 'vbr': formats_info[0].get('vbr'),
1126 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1127 'acodec': formats_info[1].get('acodec'),
1128 'abr': formats_info[1].get('abr'),
1129 'ext': output_ext,
1130 }
1131 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1132
1133 def selector_function(formats):
1134 formats = list(formats)
1135 for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1136 yield _merge(pair)
1137
1138 filters = [self._build_format_filter(f) for f in selector.filters]
1139
1140 def final_selector(formats):
1141 for _filter in filters:
1142 formats = list(filter(_filter, formats))
1143 return selector_function(formats)
1144 return final_selector
1145
1146 stream = io.BytesIO(format_spec.encode('utf-8'))
1147 try:
1148 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1149 except tokenize.TokenError:
1150 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1151
1152 class TokenIterator(object):
1153 def __init__(self, tokens):
1154 self.tokens = tokens
1155 self.counter = 0
1156
1157 def __iter__(self):
1158 return self
1159
1160 def __next__(self):
1161 if self.counter >= len(self.tokens):
1162 raise StopIteration()
1163 value = self.tokens[self.counter]
1164 self.counter += 1
1165 return value
1166
1167 next = __next__
1168
1169 def restore_last_token(self):
1170 self.counter -= 1
1171
1172 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1173 return _build_selector_function(parsed_selector)
1174
1175 def _calc_headers(self, info_dict):
1176 res = std_headers.copy()
1177
1178 add_headers = info_dict.get('http_headers')
1179 if add_headers:
1180 res.update(add_headers)
1181
1182 cookies = self._calc_cookies(info_dict)
1183 if cookies:
1184 res['Cookie'] = cookies
1185
1186 return res
1187
1188 def _calc_cookies(self, info_dict):
1189 pr = compat_urllib_request.Request(info_dict['url'])
1190 self.cookiejar.add_cookie_header(pr)
1191 return pr.get_header('Cookie')
1192
1193 def process_video_result(self, info_dict, download=True):
1194 assert info_dict.get('_type', 'video') == 'video'
1195
1196 if 'id' not in info_dict:
1197 raise ExtractorError('Missing "id" field in extractor result')
1198 if 'title' not in info_dict:
1199 raise ExtractorError('Missing "title" field in extractor result')
1200
1201 if 'playlist' not in info_dict:
1202 # It isn't part of a playlist
1203 info_dict['playlist'] = None
1204 info_dict['playlist_index'] = None
1205
1206 thumbnails = info_dict.get('thumbnails')
1207 if thumbnails is None:
1208 thumbnail = info_dict.get('thumbnail')
1209 if thumbnail:
1210 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1211 if thumbnails:
1212 thumbnails.sort(key=lambda t: (
1213 t.get('preference'), t.get('width'), t.get('height'),
1214 t.get('id'), t.get('url')))
1215 for i, t in enumerate(thumbnails):
1216 if t.get('width') and t.get('height'):
1217 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1218 if t.get('id') is None:
1219 t['id'] = '%d' % i
1220
1221 if thumbnails and 'thumbnail' not in info_dict:
1222 info_dict['thumbnail'] = thumbnails[-1]['url']
1223
1224 if 'display_id' not in info_dict and 'id' in info_dict:
1225 info_dict['display_id'] = info_dict['id']
1226
1227 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1228 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1229 # see http://bugs.python.org/issue1646728)
1230 try:
1231 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1232 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1233 except (ValueError, OverflowError, OSError):
1234 pass
1235
1236 subtitles = info_dict.get('subtitles')
1237 if subtitles:
1238 for _, subtitle in subtitles.items():
1239 for subtitle_format in subtitle:
1240 if 'ext' not in subtitle_format:
1241 subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1242
1243 if self.params.get('listsubtitles', False):
1244 if 'automatic_captions' in info_dict:
1245 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1246 self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1247 return
1248 info_dict['requested_subtitles'] = self.process_subtitles(
1249 info_dict['id'], subtitles,
1250 info_dict.get('automatic_captions'))
1251
1252 # We now pick which formats have to be downloaded
1253 if info_dict.get('formats') is None:
1254 # There's only one format available
1255 formats = [info_dict]
1256 else:
1257 formats = info_dict['formats']
1258
1259 if not formats:
1260 raise ExtractorError('No video formats found!')
1261
1262 formats_dict = {}
1263
1264 # We check that all the formats have the format and format_id fields
1265 for i, format in enumerate(formats):
1266 if 'url' not in format:
1267 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1268
1269 if format.get('format_id') is None:
1270 format['format_id'] = compat_str(i)
1271 format_id = format['format_id']
1272 if format_id not in formats_dict:
1273 formats_dict[format_id] = []
1274 formats_dict[format_id].append(format)
1275
1276 # Make sure all formats have unique format_id
1277 for format_id, ambiguous_formats in formats_dict.items():
1278 if len(ambiguous_formats) > 1:
1279 for i, format in enumerate(ambiguous_formats):
1280 format['format_id'] = '%s-%d' % (format_id, i)
1281
1282 for i, format in enumerate(formats):
1283 if format.get('format') is None:
1284 format['format'] = '{id} - {res}{note}'.format(
1285 id=format['format_id'],
1286 res=self.format_resolution(format),
1287 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1288 )
1289 # Automatically determine file extension if missing
1290 if 'ext' not in format:
1291 format['ext'] = determine_ext(format['url']).lower()
1292 # Add HTTP headers, so that external programs can use them from the
1293 # json output
1294 full_format_info = info_dict.copy()
1295 full_format_info.update(format)
1296 format['http_headers'] = self._calc_headers(full_format_info)
1297
1298 # TODO Central sorting goes here
1299
1300 if formats[0] is not info_dict:
1301 # only set the 'formats' fields if the original info_dict list them
1302 # otherwise we end up with a circular reference, the first (and unique)
1303 # element in the 'formats' field in info_dict is info_dict itself,
1304 # wich can't be exported to json
1305 info_dict['formats'] = formats
1306 if self.params.get('listformats'):
1307 self.list_formats(info_dict)
1308 return
1309 if self.params.get('list_thumbnails'):
1310 self.list_thumbnails(info_dict)
1311 return
1312
1313 req_format = self.params.get('format')
1314 if req_format is None:
1315 req_format_list = []
1316 if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1317 info_dict['extractor'] in ['youtube', 'ted'] and
1318 not info_dict.get('is_live')):
1319 merger = FFmpegMergerPP(self)
1320 if merger.available and merger.can_merge():
1321 req_format_list.append('bestvideo+bestaudio')
1322 req_format_list.append('best')
1323 req_format = '/'.join(req_format_list)
1324 format_selector = self.build_format_selector(req_format)
1325 formats_to_download = list(format_selector(formats))
1326 if not formats_to_download:
1327 raise ExtractorError('requested format not available',
1328 expected=True)
1329
1330 if download:
1331 if len(formats_to_download) > 1:
1332 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1333 for format in formats_to_download:
1334 new_info = dict(info_dict)
1335 new_info.update(format)
1336 self.process_info(new_info)
1337 # We update the info dict with the best quality format (backwards compatibility)
1338 info_dict.update(formats_to_download[-1])
1339 return info_dict
1340
1341 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1342 """Select the requested subtitles and their format"""
1343 available_subs = {}
1344 if normal_subtitles and self.params.get('writesubtitles'):
1345 available_subs.update(normal_subtitles)
1346 if automatic_captions and self.params.get('writeautomaticsub'):
1347 for lang, cap_info in automatic_captions.items():
1348 if lang not in available_subs:
1349 available_subs[lang] = cap_info
1350
1351 if (not self.params.get('writesubtitles') and not
1352 self.params.get('writeautomaticsub') or not
1353 available_subs):
1354 return None
1355
1356 if self.params.get('allsubtitles', False):
1357 requested_langs = available_subs.keys()
1358 else:
1359 if self.params.get('subtitleslangs', False):
1360 requested_langs = self.params.get('subtitleslangs')
1361 elif 'en' in available_subs:
1362 requested_langs = ['en']
1363 else:
1364 requested_langs = [list(available_subs.keys())[0]]
1365
1366 formats_query = self.params.get('subtitlesformat', 'best')
1367 formats_preference = formats_query.split('/') if formats_query else []
1368 subs = {}
1369 for lang in requested_langs:
1370 formats = available_subs.get(lang)
1371 if formats is None:
1372 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1373 continue
1374 for ext in formats_preference:
1375 if ext == 'best':
1376 f = formats[-1]
1377 break
1378 matches = list(filter(lambda f: f['ext'] == ext, formats))
1379 if matches:
1380 f = matches[-1]
1381 break
1382 else:
1383 f = formats[-1]
1384 self.report_warning(
1385 'No subtitle format found matching "%s" for language %s, '
1386 'using %s' % (formats_query, lang, f['ext']))
1387 subs[lang] = f
1388 return subs
1389
1390 def process_info(self, info_dict):
1391 """Process a single resolved IE result."""
1392
1393 assert info_dict.get('_type', 'video') == 'video'
1394
1395 max_downloads = self.params.get('max_downloads')
1396 if max_downloads is not None:
1397 if self._num_downloads >= int(max_downloads):
1398 raise MaxDownloadsReached()
1399
1400 info_dict['fulltitle'] = info_dict['title']
1401 if len(info_dict['title']) > 200:
1402 info_dict['title'] = info_dict['title'][:197] + '...'
1403
1404 if 'format' not in info_dict:
1405 info_dict['format'] = info_dict['ext']
1406
1407 reason = self._match_entry(info_dict, incomplete=False)
1408 if reason is not None:
1409 self.to_screen('[download] ' + reason)
1410 return
1411
1412 self._num_downloads += 1
1413
1414 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1415
1416 # Forced printings
1417 if self.params.get('forcetitle', False):
1418 self.to_stdout(info_dict['fulltitle'])
1419 if self.params.get('forceid', False):
1420 self.to_stdout(info_dict['id'])
1421 if self.params.get('forceurl', False):
1422 if info_dict.get('requested_formats') is not None:
1423 for f in info_dict['requested_formats']:
1424 self.to_stdout(f['url'] + f.get('play_path', ''))
1425 else:
1426 # For RTMP URLs, also include the playpath
1427 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1428 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1429 self.to_stdout(info_dict['thumbnail'])
1430 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1431 self.to_stdout(info_dict['description'])
1432 if self.params.get('forcefilename', False) and filename is not None:
1433 self.to_stdout(filename)
1434 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1435 self.to_stdout(formatSeconds(info_dict['duration']))
1436 if self.params.get('forceformat', False):
1437 self.to_stdout(info_dict['format'])
1438 if self.params.get('forcejson', False):
1439 self.to_stdout(json.dumps(info_dict))
1440
1441 # Do nothing else if in simulate mode
1442 if self.params.get('simulate', False):
1443 return
1444
1445 if filename is None:
1446 return
1447
1448 try:
1449 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1450 if dn and not os.path.exists(dn):
1451 os.makedirs(dn)
1452 except (OSError, IOError) as err:
1453 self.report_error('unable to create directory ' + compat_str(err))
1454 return
1455
1456 if self.params.get('writedescription', False):
1457 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1458 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1459 self.to_screen('[info] Video description is already present')
1460 elif info_dict.get('description') is None:
1461 self.report_warning('There\'s no description to write.')
1462 else:
1463 try:
1464 self.to_screen('[info] Writing video description to: ' + descfn)
1465 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1466 descfile.write(info_dict['description'])
1467 except (OSError, IOError):
1468 self.report_error('Cannot write description file ' + descfn)
1469 return
1470
1471 if self.params.get('writeannotations', False):
1472 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1473 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1474 self.to_screen('[info] Video annotations are already present')
1475 else:
1476 try:
1477 self.to_screen('[info] Writing video annotations to: ' + annofn)
1478 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1479 annofile.write(info_dict['annotations'])
1480 except (KeyError, TypeError):
1481 self.report_warning('There are no annotations to write.')
1482 except (OSError, IOError):
1483 self.report_error('Cannot write annotations file: ' + annofn)
1484 return
1485
1486 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1487 self.params.get('writeautomaticsub')])
1488
1489 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1490 # subtitles download errors are already managed as troubles in relevant IE
1491 # that way it will silently go on when used with unsupporting IE
1492 subtitles = info_dict['requested_subtitles']
1493 ie = self.get_info_extractor(info_dict['extractor_key'])
1494 for sub_lang, sub_info in subtitles.items():
1495 sub_format = sub_info['ext']
1496 if sub_info.get('data') is not None:
1497 sub_data = sub_info['data']
1498 else:
1499 try:
1500 sub_data = ie._download_webpage(
1501 sub_info['url'], info_dict['id'], note=False)
1502 except ExtractorError as err:
1503 self.report_warning('Unable to download subtitle for "%s": %s' %
1504 (sub_lang, compat_str(err.cause)))
1505 continue
1506 try:
1507 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1508 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1509 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1510 else:
1511 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1512 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1513 subfile.write(sub_data)
1514 except (OSError, IOError):
1515 self.report_error('Cannot write subtitles file ' + sub_filename)
1516 return
1517
1518 if self.params.get('writeinfojson', False):
1519 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1520 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1521 self.to_screen('[info] Video description metadata is already present')
1522 else:
1523 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1524 try:
1525 write_json_file(self.filter_requested_info(info_dict), infofn)
1526 except (OSError, IOError):
1527 self.report_error('Cannot write metadata to JSON file ' + infofn)
1528 return
1529
1530 self._write_thumbnails(info_dict, filename)
1531
1532 if not self.params.get('skip_download', False):
1533 try:
1534 def dl(name, info):
1535 fd = get_suitable_downloader(info, self.params)(self, self.params)
1536 for ph in self._progress_hooks:
1537 fd.add_progress_hook(ph)
1538 if self.params.get('verbose'):
1539 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1540 return fd.download(name, info)
1541
1542 if info_dict.get('requested_formats') is not None:
1543 downloaded = []
1544 success = True
1545 merger = FFmpegMergerPP(self)
1546 if not merger.available:
1547 postprocessors = []
1548 self.report_warning('You have requested multiple '
1549 'formats but ffmpeg or avconv are not installed.'
1550 ' The formats won\'t be merged.')
1551 else:
1552 postprocessors = [merger]
1553
1554 def compatible_formats(formats):
1555 video, audio = formats
1556 # Check extension
1557 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1558 if video_ext and audio_ext:
1559 COMPATIBLE_EXTS = (
1560 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1561 ('webm')
1562 )
1563 for exts in COMPATIBLE_EXTS:
1564 if video_ext in exts and audio_ext in exts:
1565 return True
1566 # TODO: Check acodec/vcodec
1567 return False
1568
1569 filename_real_ext = os.path.splitext(filename)[1][1:]
1570 filename_wo_ext = (
1571 os.path.splitext(filename)[0]
1572 if filename_real_ext == info_dict['ext']
1573 else filename)
1574 requested_formats = info_dict['requested_formats']
1575 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1576 info_dict['ext'] = 'mkv'
1577 self.report_warning(
1578 'Requested formats are incompatible for merge and will be merged into mkv.')
1579 # Ensure filename always has a correct extension for successful merge
1580 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1581 if os.path.exists(encodeFilename(filename)):
1582 self.to_screen(
1583 '[download] %s has already been downloaded and '
1584 'merged' % filename)
1585 else:
1586 for f in requested_formats:
1587 new_info = dict(info_dict)
1588 new_info.update(f)
1589 fname = self.prepare_filename(new_info)
1590 fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1591 downloaded.append(fname)
1592 partial_success = dl(fname, new_info)
1593 success = success and partial_success
1594 info_dict['__postprocessors'] = postprocessors
1595 info_dict['__files_to_merge'] = downloaded
1596 else:
1597 # Just a single file
1598 success = dl(filename, info_dict)
1599 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1600 self.report_error('unable to download video data: %s' % str(err))
1601 return
1602 except (OSError, IOError) as err:
1603 raise UnavailableVideoError(err)
1604 except (ContentTooShortError, ) as err:
1605 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1606 return
1607
1608 if success:
1609 # Fixup content
1610 fixup_policy = self.params.get('fixup')
1611 if fixup_policy is None:
1612 fixup_policy = 'detect_or_warn'
1613
1614 stretched_ratio = info_dict.get('stretched_ratio')
1615 if stretched_ratio is not None and stretched_ratio != 1:
1616 if fixup_policy == 'warn':
1617 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1618 info_dict['id'], stretched_ratio))
1619 elif fixup_policy == 'detect_or_warn':
1620 stretched_pp = FFmpegFixupStretchedPP(self)
1621 if stretched_pp.available:
1622 info_dict.setdefault('__postprocessors', [])
1623 info_dict['__postprocessors'].append(stretched_pp)
1624 else:
1625 self.report_warning(
1626 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1627 info_dict['id'], stretched_ratio))
1628 else:
1629 assert fixup_policy in ('ignore', 'never')
1630
1631 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1632 if fixup_policy == 'warn':
1633 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1634 info_dict['id']))
1635 elif fixup_policy == 'detect_or_warn':
1636 fixup_pp = FFmpegFixupM4aPP(self)
1637 if fixup_pp.available:
1638 info_dict.setdefault('__postprocessors', [])
1639 info_dict['__postprocessors'].append(fixup_pp)
1640 else:
1641 self.report_warning(
1642 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1643 info_dict['id']))
1644 else:
1645 assert fixup_policy in ('ignore', 'never')
1646
1647 try:
1648 self.post_process(filename, info_dict)
1649 except (PostProcessingError) as err:
1650 self.report_error('postprocessing: %s' % str(err))
1651 return
1652 self.record_download_archive(info_dict)
1653
1654 def download(self, url_list):
1655 """Download a given list of URLs."""
1656 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1657 if (len(url_list) > 1 and
1658 '%' not in outtmpl and
1659 self.params.get('max_downloads') != 1):
1660 raise SameFileError(outtmpl)
1661
1662 for url in url_list:
1663 try:
1664 # It also downloads the videos
1665 res = self.extract_info(
1666 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1667 except UnavailableVideoError:
1668 self.report_error('unable to download video')
1669 except MaxDownloadsReached:
1670 self.to_screen('[info] Maximum number of downloaded files reached.')
1671 raise
1672 else:
1673 if self.params.get('dump_single_json', False):
1674 self.to_stdout(json.dumps(res))
1675
1676 return self._download_retcode
1677
1678 def download_with_info_file(self, info_filename):
1679 with contextlib.closing(fileinput.FileInput(
1680 [info_filename], mode='r',
1681 openhook=fileinput.hook_encoded('utf-8'))) as f:
1682 # FileInput doesn't have a read method, we can't call json.load
1683 info = self.filter_requested_info(json.loads('\n'.join(f)))
1684 try:
1685 self.process_ie_result(info, download=True)
1686 except DownloadError:
1687 webpage_url = info.get('webpage_url')
1688 if webpage_url is not None:
1689 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1690 return self.download([webpage_url])
1691 else:
1692 raise
1693 return self._download_retcode
1694
1695 @staticmethod
1696 def filter_requested_info(info_dict):
1697 return dict(
1698 (k, v) for k, v in info_dict.items()
1699 if k not in ['requested_formats', 'requested_subtitles'])
1700
1701 def post_process(self, filename, ie_info):
1702 """Run all the postprocessors on the given file."""
1703 info = dict(ie_info)
1704 info['filepath'] = filename
1705 pps_chain = []
1706 if ie_info.get('__postprocessors') is not None:
1707 pps_chain.extend(ie_info['__postprocessors'])
1708 pps_chain.extend(self._pps)
1709 for pp in pps_chain:
1710 files_to_delete = []
1711 try:
1712 files_to_delete, info = pp.run(info)
1713 except PostProcessingError as e:
1714 self.report_error(e.msg)
1715 if files_to_delete and not self.params.get('keepvideo', False):
1716 for old_filename in files_to_delete:
1717 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1718 try:
1719 os.remove(encodeFilename(old_filename))
1720 except (IOError, OSError):
1721 self.report_warning('Unable to remove downloaded original file')
1722
1723 def _make_archive_id(self, info_dict):
1724 # Future-proof against any change in case
1725 # and backwards compatibility with prior versions
1726 extractor = info_dict.get('extractor_key')
1727 if extractor is None:
1728 if 'id' in info_dict:
1729 extractor = info_dict.get('ie_key') # key in a playlist
1730 if extractor is None:
1731 return None # Incomplete video information
1732 return extractor.lower() + ' ' + info_dict['id']
1733
1734 def in_download_archive(self, info_dict):
1735 fn = self.params.get('download_archive')
1736 if fn is None:
1737 return False
1738
1739 vid_id = self._make_archive_id(info_dict)
1740 if vid_id is None:
1741 return False # Incomplete video information
1742
1743 try:
1744 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1745 for line in archive_file:
1746 if line.strip() == vid_id:
1747 return True
1748 except IOError as ioe:
1749 if ioe.errno != errno.ENOENT:
1750 raise
1751 return False
1752
1753 def record_download_archive(self, info_dict):
1754 fn = self.params.get('download_archive')
1755 if fn is None:
1756 return
1757 vid_id = self._make_archive_id(info_dict)
1758 assert vid_id
1759 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1760 archive_file.write(vid_id + '\n')
1761
1762 @staticmethod
1763 def format_resolution(format, default='unknown'):
1764 if format.get('vcodec') == 'none':
1765 return 'audio only'
1766 if format.get('resolution') is not None:
1767 return format['resolution']
1768 if format.get('height') is not None:
1769 if format.get('width') is not None:
1770 res = '%sx%s' % (format['width'], format['height'])
1771 else:
1772 res = '%sp' % format['height']
1773 elif format.get('width') is not None:
1774 res = '?x%d' % format['width']
1775 else:
1776 res = default
1777 return res
1778
1779 def _format_note(self, fdict):
1780 res = ''
1781 if fdict.get('ext') in ['f4f', 'f4m']:
1782 res += '(unsupported) '
1783 if fdict.get('format_note') is not None:
1784 res += fdict['format_note'] + ' '
1785 if fdict.get('tbr') is not None:
1786 res += '%4dk ' % fdict['tbr']
1787 if fdict.get('container') is not None:
1788 if res:
1789 res += ', '
1790 res += '%s container' % fdict['container']
1791 if (fdict.get('vcodec') is not None and
1792 fdict.get('vcodec') != 'none'):
1793 if res:
1794 res += ', '
1795 res += fdict['vcodec']
1796 if fdict.get('vbr') is not None:
1797 res += '@'
1798 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1799 res += 'video@'
1800 if fdict.get('vbr') is not None:
1801 res += '%4dk' % fdict['vbr']
1802 if fdict.get('fps') is not None:
1803 res += ', %sfps' % fdict['fps']
1804 if fdict.get('acodec') is not None:
1805 if res:
1806 res += ', '
1807 if fdict['acodec'] == 'none':
1808 res += 'video only'
1809 else:
1810 res += '%-5s' % fdict['acodec']
1811 elif fdict.get('abr') is not None:
1812 if res:
1813 res += ', '
1814 res += 'audio'
1815 if fdict.get('abr') is not None:
1816 res += '@%3dk' % fdict['abr']
1817 if fdict.get('asr') is not None:
1818 res += ' (%5dHz)' % fdict['asr']
1819 if fdict.get('filesize') is not None:
1820 if res:
1821 res += ', '
1822 res += format_bytes(fdict['filesize'])
1823 elif fdict.get('filesize_approx') is not None:
1824 if res:
1825 res += ', '
1826 res += '~' + format_bytes(fdict['filesize_approx'])
1827 return res
1828
1829 def list_formats(self, info_dict):
1830 formats = info_dict.get('formats', [info_dict])
1831 table = [
1832 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1833 for f in formats
1834 if f.get('preference') is None or f['preference'] >= -1000]
1835 if len(formats) > 1:
1836 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1837
1838 header_line = ['format code', 'extension', 'resolution', 'note']
1839 self.to_screen(
1840 '[info] Available formats for %s:\n%s' %
1841 (info_dict['id'], render_table(header_line, table)))
1842
1843 def list_thumbnails(self, info_dict):
1844 thumbnails = info_dict.get('thumbnails')
1845 if not thumbnails:
1846 tn_url = info_dict.get('thumbnail')
1847 if tn_url:
1848 thumbnails = [{'id': '0', 'url': tn_url}]
1849 else:
1850 self.to_screen(
1851 '[info] No thumbnails present for %s' % info_dict['id'])
1852 return
1853
1854 self.to_screen(
1855 '[info] Thumbnails for %s:' % info_dict['id'])
1856 self.to_screen(render_table(
1857 ['ID', 'width', 'height', 'URL'],
1858 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1859
1860 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1861 if not subtitles:
1862 self.to_screen('%s has no %s' % (video_id, name))
1863 return
1864 self.to_screen(
1865 'Available %s for %s:' % (name, video_id))
1866 self.to_screen(render_table(
1867 ['Language', 'formats'],
1868 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1869 for lang, formats in subtitles.items()]))
1870
1871 def urlopen(self, req):
1872 """ Start an HTTP download """
1873 return self._opener.open(req, timeout=self._socket_timeout)
1874
1875 def print_debug_header(self):
1876 if not self.params.get('verbose'):
1877 return
1878
1879 if type('') is not compat_str:
1880 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1881 self.report_warning(
1882 'Your Python is broken! Update to a newer and supported version')
1883
1884 stdout_encoding = getattr(
1885 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1886 encoding_str = (
1887 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1888 locale.getpreferredencoding(),
1889 sys.getfilesystemencoding(),
1890 stdout_encoding,
1891 self.get_encoding()))
1892 write_string(encoding_str, encoding=None)
1893
1894 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1895 try:
1896 sp = subprocess.Popen(
1897 ['git', 'rev-parse', '--short', 'HEAD'],
1898 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1899 cwd=os.path.dirname(os.path.abspath(__file__)))
1900 out, err = sp.communicate()
1901 out = out.decode().strip()
1902 if re.match('[0-9a-f]+', out):
1903 self._write_string('[debug] Git HEAD: ' + out + '\n')
1904 except Exception:
1905 try:
1906 sys.exc_clear()
1907 except Exception:
1908 pass
1909 self._write_string('[debug] Python version %s - %s\n' % (
1910 platform.python_version(), platform_name()))
1911
1912 exe_versions = FFmpegPostProcessor.get_versions(self)
1913 exe_versions['rtmpdump'] = rtmpdump_version()
1914 exe_str = ', '.join(
1915 '%s %s' % (exe, v)
1916 for exe, v in sorted(exe_versions.items())
1917 if v
1918 )
1919 if not exe_str:
1920 exe_str = 'none'
1921 self._write_string('[debug] exe versions: %s\n' % exe_str)
1922
1923 proxy_map = {}
1924 for handler in self._opener.handlers:
1925 if hasattr(handler, 'proxies'):
1926 proxy_map.update(handler.proxies)
1927 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1928
1929 if self.params.get('call_home', False):
1930 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1931 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1932 latest_version = self.urlopen(
1933 'https://yt-dl.org/latest/version').read().decode('utf-8')
1934 if version_tuple(latest_version) > version_tuple(__version__):
1935 self.report_warning(
1936 'You are using an outdated version (newest version: %s)! '
1937 'See https://yt-dl.org/update if you need help updating.' %
1938 latest_version)
1939
1940 def _setup_opener(self):
1941 timeout_val = self.params.get('socket_timeout')
1942 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1943
1944 opts_cookiefile = self.params.get('cookiefile')
1945 opts_proxy = self.params.get('proxy')
1946
1947 if opts_cookiefile is None:
1948 self.cookiejar = compat_cookiejar.CookieJar()
1949 else:
1950 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1951 opts_cookiefile)
1952 if os.access(opts_cookiefile, os.R_OK):
1953 self.cookiejar.load()
1954
1955 cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
1956 if opts_proxy is not None:
1957 if opts_proxy == '':
1958 proxies = {}
1959 else:
1960 proxies = {'http': opts_proxy, 'https': opts_proxy}
1961 else:
1962 proxies = compat_urllib_request.getproxies()
1963 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1964 if 'http' in proxies and 'https' not in proxies:
1965 proxies['https'] = proxies['http']
1966 proxy_handler = PerRequestProxyHandler(proxies)
1967
1968 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1969 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1970 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1971 data_handler = compat_urllib_request_DataHandler()
1972 opener = compat_urllib_request.build_opener(
1973 proxy_handler, https_handler, cookie_processor, ydlh, data_handler)
1974
1975 # Delete the default user-agent header, which would otherwise apply in
1976 # cases where our custom HTTP handler doesn't come into play
1977 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1978 opener.addheaders = []
1979 self._opener = opener
1980
1981 def encode(self, s):
1982 if isinstance(s, bytes):
1983 return s # Already encoded
1984
1985 try:
1986 return s.encode(self.get_encoding())
1987 except UnicodeEncodeError as err:
1988 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1989 raise
1990
1991 def get_encoding(self):
1992 encoding = self.params.get('encoding')
1993 if encoding is None:
1994 encoding = preferredencoding()
1995 return encoding
1996
1997 def _write_thumbnails(self, info_dict, filename):
1998 if self.params.get('writethumbnail', False):
1999 thumbnails = info_dict.get('thumbnails')
2000 if thumbnails:
2001 thumbnails = [thumbnails[-1]]
2002 elif self.params.get('write_all_thumbnails', False):
2003 thumbnails = info_dict.get('thumbnails')
2004 else:
2005 return
2006
2007 if not thumbnails:
2008 # No thumbnails present, so return immediately
2009 return
2010
2011 for t in thumbnails:
2012 thumb_ext = determine_ext(t['url'], 'jpg')
2013 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2014 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2015 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2016
2017 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2018 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2019 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2020 else:
2021 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2022 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2023 try:
2024 uf = self.urlopen(t['url'])
2025 with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2026 shutil.copyfileobj(uf, thumbf)
2027 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2028 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2029 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2030 self.report_warning('Unable to download thumbnail "%s": %s' %
2031 (t['url'], compat_str(err)))