]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py
a7bf5a1b06766094cc06e85fd40c6fa64c2cc64b
[youtubedl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 from __future__ import absolute_import, unicode_literals
5
6 import collections
7 import contextlib
8 import copy
9 import datetime
10 import errno
11 import fileinput
12 import io
13 import itertools
14 import json
15 import locale
16 import operator
17 import os
18 import platform
19 import re
20 import shutil
21 import subprocess
22 import socket
23 import sys
24 import time
25 import tokenize
26 import traceback
27 import random
28
29 from .compat import (
30 compat_basestring,
31 compat_cookiejar,
32 compat_expanduser,
33 compat_get_terminal_size,
34 compat_http_client,
35 compat_kwargs,
36 compat_os_name,
37 compat_str,
38 compat_tokenize_tokenize,
39 compat_urllib_error,
40 compat_urllib_request,
41 compat_urllib_request_DataHandler,
42 )
43 from .utils import (
44 age_restricted,
45 args_to_str,
46 ContentTooShortError,
47 date_from_str,
48 DateRange,
49 DEFAULT_OUTTMPL,
50 determine_ext,
51 determine_protocol,
52 DownloadError,
53 encode_compat_str,
54 encodeFilename,
55 error_to_compat_str,
56 ExtractorError,
57 format_bytes,
58 formatSeconds,
59 locked_file,
60 make_HTTPS_handler,
61 MaxDownloadsReached,
62 PagedList,
63 parse_filesize,
64 PerRequestProxyHandler,
65 platform_name,
66 PostProcessingError,
67 preferredencoding,
68 prepend_extension,
69 register_socks_protocols,
70 render_table,
71 replace_extension,
72 SameFileError,
73 sanitize_filename,
74 sanitize_path,
75 sanitize_url,
76 sanitized_Request,
77 std_headers,
78 subtitles_filename,
79 UnavailableVideoError,
80 url_basename,
81 version_tuple,
82 write_json_file,
83 write_string,
84 YoutubeDLCookieProcessor,
85 YoutubeDLHandler,
86 )
87 from .cache import Cache
88 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
89 from .downloader import get_suitable_downloader
90 from .downloader.rtmp import rtmpdump_version
91 from .postprocessor import (
92 FFmpegFixupM3u8PP,
93 FFmpegFixupM4aPP,
94 FFmpegFixupStretchedPP,
95 FFmpegMergerPP,
96 FFmpegPostProcessor,
97 get_postprocessor,
98 )
99 from .version import __version__
100
101 if compat_os_name == 'nt':
102 import ctypes
103
104
105 class YoutubeDL(object):
106 """YoutubeDL class.
107
108 YoutubeDL objects are the ones responsible of downloading the
109 actual video file and writing it to disk if the user has requested
110 it, among some other tasks. In most cases there should be one per
111 program. As, given a video URL, the downloader doesn't know how to
112 extract all the needed information, task that InfoExtractors do, it
113 has to pass the URL to one of them.
114
115 For this, YoutubeDL objects have a method that allows
116 InfoExtractors to be registered in a given order. When it is passed
117 a URL, the YoutubeDL object handles it to the first InfoExtractor it
118 finds that reports being able to handle it. The InfoExtractor extracts
119 all the information about the video or videos the URL refers to, and
120 YoutubeDL process the extracted information, possibly using a File
121 Downloader to download the video.
122
123 YoutubeDL objects accept a lot of parameters. In order not to saturate
124 the object constructor with arguments, it receives a dictionary of
125 options instead. These options are available through the params
126 attribute for the InfoExtractors to use. The YoutubeDL also
127 registers itself as the downloader in charge for the InfoExtractors
128 that are added to it, so this is a "mutual registration".
129
130 Available options:
131
132 username: Username for authentication purposes.
133 password: Password for authentication purposes.
134 videopassword: Password for accessing a video.
135 ap_mso: Adobe Pass multiple-system operator identifier.
136 ap_username: Multiple-system operator account username.
137 ap_password: Multiple-system operator account password.
138 usenetrc: Use netrc for authentication instead.
139 verbose: Print additional info to stdout.
140 quiet: Do not print messages to stdout.
141 no_warnings: Do not print out anything for warnings.
142 forceurl: Force printing final URL.
143 forcetitle: Force printing title.
144 forceid: Force printing ID.
145 forcethumbnail: Force printing thumbnail URL.
146 forcedescription: Force printing description.
147 forcefilename: Force printing final filename.
148 forceduration: Force printing duration.
149 forcejson: Force printing info_dict as JSON.
150 dump_single_json: Force printing the info_dict of the whole playlist
151 (or video) as a single JSON line.
152 simulate: Do not download the video files.
153 format: Video format code. See options.py for more information.
154 outtmpl: Template for output names.
155 restrictfilenames: Do not allow "&" and spaces in file names
156 ignoreerrors: Do not stop on download errors.
157 force_generic_extractor: Force downloader to use the generic extractor
158 nooverwrites: Prevent overwriting files.
159 playliststart: Playlist item to start at.
160 playlistend: Playlist item to end at.
161 playlist_items: Specific indices of playlist to download.
162 playlistreverse: Download playlist items in reverse order.
163 playlistrandom: Download playlist items in random order.
164 matchtitle: Download only matching titles.
165 rejecttitle: Reject downloads for matching titles.
166 logger: Log messages to a logging.Logger instance.
167 logtostderr: Log messages to stderr instead of stdout.
168 writedescription: Write the video description to a .description file
169 writeinfojson: Write the video description to a .info.json file
170 writeannotations: Write the video annotations to a .annotations.xml file
171 writethumbnail: Write the thumbnail image to a file
172 write_all_thumbnails: Write all thumbnail formats to files
173 writesubtitles: Write the video subtitles to a file
174 writeautomaticsub: Write the automatically generated subtitles to a file
175 allsubtitles: Downloads all the subtitles of the video
176 (requires writesubtitles or writeautomaticsub)
177 listsubtitles: Lists all available subtitles for the video
178 subtitlesformat: The format code for subtitles
179 subtitleslangs: List of languages of the subtitles to download
180 keepvideo: Keep the video file after post-processing
181 daterange: A DateRange object, download only if the upload_date is in the range.
182 skip_download: Skip the actual download of the video file
183 cachedir: Location of the cache files in the filesystem.
184 False to disable filesystem cache.
185 noplaylist: Download single video instead of a playlist if in doubt.
186 age_limit: An integer representing the user's age in years.
187 Unsuitable videos for the given age are skipped.
188 min_views: An integer representing the minimum view count the video
189 must have in order to not be skipped.
190 Videos without view count information are always
191 downloaded. None for no limit.
192 max_views: An integer representing the maximum view count.
193 Videos that are more popular than that are not
194 downloaded.
195 Videos without view count information are always
196 downloaded. None for no limit.
197 download_archive: File name of a file where all downloads are recorded.
198 Videos already present in the file are not downloaded
199 again.
200 cookiefile: File name where cookies should be read from and dumped to.
201 nocheckcertificate:Do not verify SSL certificates
202 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
203 At the moment, this is only supported by YouTube.
204 proxy: URL of the proxy server to use
205 geo_verification_proxy: URL of the proxy to use for IP address verification
206 on geo-restricted sites. (Experimental)
207 socket_timeout: Time to wait for unresponsive hosts, in seconds
208 bidi_workaround: Work around buggy terminals without bidirectional text
209 support, using fridibi
210 debug_printtraffic:Print out sent and received HTTP traffic
211 include_ads: Download ads as well
212 default_search: Prepend this string if an input url is not valid.
213 'auto' for elaborate guessing
214 encoding: Use this encoding instead of the system-specified.
215 extract_flat: Do not resolve URLs, return the immediate result.
216 Pass in 'in_playlist' to only show this behavior for
217 playlist items.
218 postprocessors: A list of dictionaries, each with an entry
219 * key: The name of the postprocessor. See
220 youtube_dl/postprocessor/__init__.py for a list.
221 as well as any further keyword arguments for the
222 postprocessor.
223 progress_hooks: A list of functions that get called on download
224 progress, with a dictionary with the entries
225 * status: One of "downloading", "error", or "finished".
226 Check this first and ignore unknown values.
227
228 If status is one of "downloading", or "finished", the
229 following properties may also be present:
230 * filename: The final filename (always present)
231 * tmpfilename: The filename we're currently writing to
232 * downloaded_bytes: Bytes on disk
233 * total_bytes: Size of the whole file, None if unknown
234 * total_bytes_estimate: Guess of the eventual file size,
235 None if unavailable.
236 * elapsed: The number of seconds since download started.
237 * eta: The estimated time in seconds, None if unknown
238 * speed: The download speed in bytes/second, None if
239 unknown
240 * fragment_index: The counter of the currently
241 downloaded video fragment.
242 * fragment_count: The number of fragments (= individual
243 files that will be merged)
244
245 Progress hooks are guaranteed to be called at least once
246 (with status "finished") if the download is successful.
247 merge_output_format: Extension to use when merging formats.
248 fixup: Automatically correct known faults of the file.
249 One of:
250 - "never": do nothing
251 - "warn": only emit a warning
252 - "detect_or_warn": check whether we can do anything
253 about it, warn otherwise (default)
254 source_address: (Experimental) Client-side IP address to bind to.
255 call_home: Boolean, true iff we are allowed to contact the
256 youtube-dl servers for debugging.
257 sleep_interval: Number of seconds to sleep before each download when
258 used alone or a lower bound of a range for randomized
259 sleep before each download (minimum possible number
260 of seconds to sleep) when used along with
261 max_sleep_interval.
262 max_sleep_interval:Upper bound of a range for randomized sleep before each
263 download (maximum possible number of seconds to sleep).
264 Must only be used along with sleep_interval.
265 Actual sleep time will be a random float from range
266 [sleep_interval; max_sleep_interval].
267 listformats: Print an overview of available video formats and exit.
268 list_thumbnails: Print a table of all thumbnails and exit.
269 match_filter: A function that gets called with the info_dict of
270 every video.
271 If it returns a message, the video is ignored.
272 If it returns None, the video is downloaded.
273 match_filter_func in utils.py is one example for this.
274 no_color: Do not emit color codes in output.
275
276 The following options determine which downloader is picked:
277 external_downloader: Executable of the external downloader to call.
278 None or unset for standard (built-in) downloader.
279 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
280 if True, otherwise use ffmpeg/avconv if False, otherwise
281 use downloader suggested by extractor if None.
282
283 The following parameters are not used by YoutubeDL itself, they are used by
284 the downloader (see youtube_dl/downloader/common.py):
285 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
286 noresizebuffer, retries, continuedl, noprogress, consoletitle,
287 xattr_set_filesize, external_downloader_args, hls_use_mpegts.
288
289 The following options are used by the post processors:
290 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
291 otherwise prefer avconv.
292 postprocessor_args: A list of additional command-line arguments for the
293 postprocessor.
294 """
295
296 params = None
297 _ies = []
298 _pps = []
299 _download_retcode = None
300 _num_downloads = None
301 _screen_file = None
302
303 def __init__(self, params=None, auto_init=True):
304 """Create a FileDownloader object with the given options."""
305 if params is None:
306 params = {}
307 self._ies = []
308 self._ies_instances = {}
309 self._pps = []
310 self._progress_hooks = []
311 self._download_retcode = 0
312 self._num_downloads = 0
313 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
314 self._err_file = sys.stderr
315 self.params = {
316 # Default parameters
317 'nocheckcertificate': False,
318 }
319 self.params.update(params)
320 self.cache = Cache(self)
321
322 if self.params.get('cn_verification_proxy') is not None:
323 self.report_warning('--cn-verification-proxy is deprecated. Use --geo-verification-proxy instead.')
324 if self.params.get('geo_verification_proxy') is None:
325 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
326
327 if params.get('bidi_workaround', False):
328 try:
329 import pty
330 master, slave = pty.openpty()
331 width = compat_get_terminal_size().columns
332 if width is None:
333 width_args = []
334 else:
335 width_args = ['-w', str(width)]
336 sp_kwargs = dict(
337 stdin=subprocess.PIPE,
338 stdout=slave,
339 stderr=self._err_file)
340 try:
341 self._output_process = subprocess.Popen(
342 ['bidiv'] + width_args, **sp_kwargs
343 )
344 except OSError:
345 self._output_process = subprocess.Popen(
346 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
347 self._output_channel = os.fdopen(master, 'rb')
348 except OSError as ose:
349 if ose.errno == errno.ENOENT:
350 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
351 else:
352 raise
353
354 if (sys.version_info >= (3,) and sys.platform != 'win32' and
355 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
356 not params.get('restrictfilenames', False)):
357 # On Python 3, the Unicode filesystem API will throw errors (#1474)
358 self.report_warning(
359 'Assuming --restrict-filenames since file system encoding '
360 'cannot encode all characters. '
361 'Set the LC_ALL environment variable to fix this.')
362 self.params['restrictfilenames'] = True
363
364 if isinstance(params.get('outtmpl'), bytes):
365 self.report_warning(
366 'Parameter outtmpl is bytes, but should be a unicode string. '
367 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
368
369 self._setup_opener()
370
371 if auto_init:
372 self.print_debug_header()
373 self.add_default_info_extractors()
374
375 for pp_def_raw in self.params.get('postprocessors', []):
376 pp_class = get_postprocessor(pp_def_raw['key'])
377 pp_def = dict(pp_def_raw)
378 del pp_def['key']
379 pp = pp_class(self, **compat_kwargs(pp_def))
380 self.add_post_processor(pp)
381
382 for ph in self.params.get('progress_hooks', []):
383 self.add_progress_hook(ph)
384
385 register_socks_protocols()
386
387 def warn_if_short_id(self, argv):
388 # short YouTube ID starting with dash?
389 idxs = [
390 i for i, a in enumerate(argv)
391 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
392 if idxs:
393 correct_argv = (
394 ['youtube-dl'] +
395 [a for i, a in enumerate(argv) if i not in idxs] +
396 ['--'] + [argv[i] for i in idxs]
397 )
398 self.report_warning(
399 'Long argument string detected. '
400 'Use -- to separate parameters and URLs, like this:\n%s\n' %
401 args_to_str(correct_argv))
402
403 def add_info_extractor(self, ie):
404 """Add an InfoExtractor object to the end of the list."""
405 self._ies.append(ie)
406 if not isinstance(ie, type):
407 self._ies_instances[ie.ie_key()] = ie
408 ie.set_downloader(self)
409
410 def get_info_extractor(self, ie_key):
411 """
412 Get an instance of an IE with name ie_key, it will try to get one from
413 the _ies list, if there's no instance it will create a new one and add
414 it to the extractor list.
415 """
416 ie = self._ies_instances.get(ie_key)
417 if ie is None:
418 ie = get_info_extractor(ie_key)()
419 self.add_info_extractor(ie)
420 return ie
421
422 def add_default_info_extractors(self):
423 """
424 Add the InfoExtractors returned by gen_extractors to the end of the list
425 """
426 for ie in gen_extractor_classes():
427 self.add_info_extractor(ie)
428
429 def add_post_processor(self, pp):
430 """Add a PostProcessor object to the end of the chain."""
431 self._pps.append(pp)
432 pp.set_downloader(self)
433
434 def add_progress_hook(self, ph):
435 """Add the progress hook (currently only for the file downloader)"""
436 self._progress_hooks.append(ph)
437
438 def _bidi_workaround(self, message):
439 if not hasattr(self, '_output_channel'):
440 return message
441
442 assert hasattr(self, '_output_process')
443 assert isinstance(message, compat_str)
444 line_count = message.count('\n') + 1
445 self._output_process.stdin.write((message + '\n').encode('utf-8'))
446 self._output_process.stdin.flush()
447 res = ''.join(self._output_channel.readline().decode('utf-8')
448 for _ in range(line_count))
449 return res[:-len('\n')]
450
451 def to_screen(self, message, skip_eol=False):
452 """Print message to stdout if not in quiet mode."""
453 return self.to_stdout(message, skip_eol, check_quiet=True)
454
455 def _write_string(self, s, out=None):
456 write_string(s, out=out, encoding=self.params.get('encoding'))
457
458 def to_stdout(self, message, skip_eol=False, check_quiet=False):
459 """Print message to stdout if not in quiet mode."""
460 if self.params.get('logger'):
461 self.params['logger'].debug(message)
462 elif not check_quiet or not self.params.get('quiet', False):
463 message = self._bidi_workaround(message)
464 terminator = ['\n', ''][skip_eol]
465 output = message + terminator
466
467 self._write_string(output, self._screen_file)
468
469 def to_stderr(self, message):
470 """Print message to stderr."""
471 assert isinstance(message, compat_str)
472 if self.params.get('logger'):
473 self.params['logger'].error(message)
474 else:
475 message = self._bidi_workaround(message)
476 output = message + '\n'
477 self._write_string(output, self._err_file)
478
479 def to_console_title(self, message):
480 if not self.params.get('consoletitle', False):
481 return
482 if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
483 # c_wchar_p() might not be necessary if `message` is
484 # already of type unicode()
485 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
486 elif 'TERM' in os.environ:
487 self._write_string('\033]0;%s\007' % message, self._screen_file)
488
489 def save_console_title(self):
490 if not self.params.get('consoletitle', False):
491 return
492 if 'TERM' in os.environ:
493 # Save the title on stack
494 self._write_string('\033[22;0t', self._screen_file)
495
496 def restore_console_title(self):
497 if not self.params.get('consoletitle', False):
498 return
499 if 'TERM' in os.environ:
500 # Restore the title from stack
501 self._write_string('\033[23;0t', self._screen_file)
502
503 def __enter__(self):
504 self.save_console_title()
505 return self
506
507 def __exit__(self, *args):
508 self.restore_console_title()
509
510 if self.params.get('cookiefile') is not None:
511 self.cookiejar.save()
512
513 def trouble(self, message=None, tb=None):
514 """Determine action to take when a download problem appears.
515
516 Depending on if the downloader has been configured to ignore
517 download errors or not, this method may throw an exception or
518 not when errors are found, after printing the message.
519
520 tb, if given, is additional traceback information.
521 """
522 if message is not None:
523 self.to_stderr(message)
524 if self.params.get('verbose'):
525 if tb is None:
526 if sys.exc_info()[0]: # if .trouble has been called from an except block
527 tb = ''
528 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
529 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
530 tb += encode_compat_str(traceback.format_exc())
531 else:
532 tb_data = traceback.format_list(traceback.extract_stack())
533 tb = ''.join(tb_data)
534 self.to_stderr(tb)
535 if not self.params.get('ignoreerrors', False):
536 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
537 exc_info = sys.exc_info()[1].exc_info
538 else:
539 exc_info = sys.exc_info()
540 raise DownloadError(message, exc_info)
541 self._download_retcode = 1
542
543 def report_warning(self, message):
544 '''
545 Print the message to stderr, it will be prefixed with 'WARNING:'
546 If stderr is a tty file the 'WARNING:' will be colored
547 '''
548 if self.params.get('logger') is not None:
549 self.params['logger'].warning(message)
550 else:
551 if self.params.get('no_warnings'):
552 return
553 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
554 _msg_header = '\033[0;33mWARNING:\033[0m'
555 else:
556 _msg_header = 'WARNING:'
557 warning_message = '%s %s' % (_msg_header, message)
558 self.to_stderr(warning_message)
559
560 def report_error(self, message, tb=None):
561 '''
562 Do the same as trouble, but prefixes the message with 'ERROR:', colored
563 in red if stderr is a tty file.
564 '''
565 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
566 _msg_header = '\033[0;31mERROR:\033[0m'
567 else:
568 _msg_header = 'ERROR:'
569 error_message = '%s %s' % (_msg_header, message)
570 self.trouble(error_message, tb)
571
572 def report_file_already_downloaded(self, file_name):
573 """Report file has already been fully downloaded."""
574 try:
575 self.to_screen('[download] %s has already been downloaded' % file_name)
576 except UnicodeEncodeError:
577 self.to_screen('[download] The file has already been downloaded')
578
579 def prepare_filename(self, info_dict):
580 """Generate the output filename."""
581 try:
582 template_dict = dict(info_dict)
583
584 template_dict['epoch'] = int(time.time())
585 autonumber_size = self.params.get('autonumber_size')
586 if autonumber_size is None:
587 autonumber_size = 5
588 autonumber_templ = '%0' + str(autonumber_size) + 'd'
589 template_dict['autonumber'] = autonumber_templ % (self.params.get('autonumber_start', 1) - 1 + self._num_downloads)
590 if template_dict.get('playlist_index') is not None:
591 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
592 if template_dict.get('resolution') is None:
593 if template_dict.get('width') and template_dict.get('height'):
594 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
595 elif template_dict.get('height'):
596 template_dict['resolution'] = '%sp' % template_dict['height']
597 elif template_dict.get('width'):
598 template_dict['resolution'] = '%dx?' % template_dict['width']
599
600 sanitize = lambda k, v: sanitize_filename(
601 compat_str(v),
602 restricted=self.params.get('restrictfilenames'),
603 is_id=(k == 'id'))
604 template_dict = dict((k, sanitize(k, v))
605 for k, v in template_dict.items()
606 if v is not None and not isinstance(v, (list, tuple, dict)))
607 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
608
609 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
610 tmpl = compat_expanduser(outtmpl)
611 filename = tmpl % template_dict
612 # Temporary fix for #4787
613 # 'Treat' all problem characters by passing filename through preferredencoding
614 # to workaround encoding issues with subprocess on python2 @ Windows
615 if sys.version_info < (3, 0) and sys.platform == 'win32':
616 filename = encodeFilename(filename, True).decode(preferredencoding())
617 return sanitize_path(filename)
618 except ValueError as err:
619 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
620 return None
621
622 def _match_entry(self, info_dict, incomplete):
623 """ Returns None iff the file should be downloaded """
624
625 video_title = info_dict.get('title', info_dict.get('id', 'video'))
626 if 'title' in info_dict:
627 # This can happen when we're just evaluating the playlist
628 title = info_dict['title']
629 matchtitle = self.params.get('matchtitle', False)
630 if matchtitle:
631 if not re.search(matchtitle, title, re.IGNORECASE):
632 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
633 rejecttitle = self.params.get('rejecttitle', False)
634 if rejecttitle:
635 if re.search(rejecttitle, title, re.IGNORECASE):
636 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
637 date = info_dict.get('upload_date')
638 if date is not None:
639 dateRange = self.params.get('daterange', DateRange())
640 if date not in dateRange:
641 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
642 view_count = info_dict.get('view_count')
643 if view_count is not None:
644 min_views = self.params.get('min_views')
645 if min_views is not None and view_count < min_views:
646 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
647 max_views = self.params.get('max_views')
648 if max_views is not None and view_count > max_views:
649 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
650 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
651 return 'Skipping "%s" because it is age restricted' % video_title
652 if self.in_download_archive(info_dict):
653 return '%s has already been recorded in archive' % video_title
654
655 if not incomplete:
656 match_filter = self.params.get('match_filter')
657 if match_filter is not None:
658 ret = match_filter(info_dict)
659 if ret is not None:
660 return ret
661
662 return None
663
664 @staticmethod
665 def add_extra_info(info_dict, extra_info):
666 '''Set the keys from extra_info in info dict if they are missing'''
667 for key, value in extra_info.items():
668 info_dict.setdefault(key, value)
669
670 def extract_info(self, url, download=True, ie_key=None, extra_info={},
671 process=True, force_generic_extractor=False):
672 '''
673 Returns a list with a dictionary for each video we find.
674 If 'download', also downloads the videos.
675 extra_info is a dict containing the extra values to add to each result
676 '''
677
678 if not ie_key and force_generic_extractor:
679 ie_key = 'Generic'
680
681 if ie_key:
682 ies = [self.get_info_extractor(ie_key)]
683 else:
684 ies = self._ies
685
686 for ie in ies:
687 if not ie.suitable(url):
688 continue
689
690 ie = self.get_info_extractor(ie.ie_key())
691 if not ie.working():
692 self.report_warning('The program functionality for this site has been marked as broken, '
693 'and will probably not work.')
694
695 try:
696 ie_result = ie.extract(url)
697 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
698 break
699 if isinstance(ie_result, list):
700 # Backwards compatibility: old IE result format
701 ie_result = {
702 '_type': 'compat_list',
703 'entries': ie_result,
704 }
705 self.add_default_extra_info(ie_result, ie, url)
706 if process:
707 return self.process_ie_result(ie_result, download, extra_info)
708 else:
709 return ie_result
710 except ExtractorError as e: # An error we somewhat expected
711 self.report_error(compat_str(e), e.format_traceback())
712 break
713 except MaxDownloadsReached:
714 raise
715 except Exception as e:
716 if self.params.get('ignoreerrors', False):
717 self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
718 break
719 else:
720 raise
721 else:
722 self.report_error('no suitable InfoExtractor for URL %s' % url)
723
724 def add_default_extra_info(self, ie_result, ie, url):
725 self.add_extra_info(ie_result, {
726 'extractor': ie.IE_NAME,
727 'webpage_url': url,
728 'webpage_url_basename': url_basename(url),
729 'extractor_key': ie.ie_key(),
730 })
731
732 def process_ie_result(self, ie_result, download=True, extra_info={}):
733 """
734 Take the result of the ie(may be modified) and resolve all unresolved
735 references (URLs, playlist items).
736
737 It will also download the videos if 'download'.
738 Returns the resolved ie_result.
739 """
740 result_type = ie_result.get('_type', 'video')
741
742 if result_type in ('url', 'url_transparent'):
743 ie_result['url'] = sanitize_url(ie_result['url'])
744 extract_flat = self.params.get('extract_flat', False)
745 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
746 extract_flat is True):
747 if self.params.get('forcejson', False):
748 self.to_stdout(json.dumps(ie_result))
749 return ie_result
750
751 if result_type == 'video':
752 self.add_extra_info(ie_result, extra_info)
753 return self.process_video_result(ie_result, download=download)
754 elif result_type == 'url':
755 # We have to add extra_info to the results because it may be
756 # contained in a playlist
757 return self.extract_info(ie_result['url'],
758 download,
759 ie_key=ie_result.get('ie_key'),
760 extra_info=extra_info)
761 elif result_type == 'url_transparent':
762 # Use the information from the embedding page
763 info = self.extract_info(
764 ie_result['url'], ie_key=ie_result.get('ie_key'),
765 extra_info=extra_info, download=False, process=False)
766
767 force_properties = dict(
768 (k, v) for k, v in ie_result.items() if v is not None)
769 for f in ('_type', 'url', 'ie_key'):
770 if f in force_properties:
771 del force_properties[f]
772 new_result = info.copy()
773 new_result.update(force_properties)
774
775 assert new_result.get('_type') != 'url_transparent'
776
777 return self.process_ie_result(
778 new_result, download=download, extra_info=extra_info)
779 elif result_type == 'playlist' or result_type == 'multi_video':
780 # We process each entry in the playlist
781 playlist = ie_result.get('title') or ie_result.get('id')
782 self.to_screen('[download] Downloading playlist: %s' % playlist)
783
784 playlist_results = []
785
786 playliststart = self.params.get('playliststart', 1) - 1
787 playlistend = self.params.get('playlistend')
788 # For backwards compatibility, interpret -1 as whole list
789 if playlistend == -1:
790 playlistend = None
791
792 playlistitems_str = self.params.get('playlist_items')
793 playlistitems = None
794 if playlistitems_str is not None:
795 def iter_playlistitems(format):
796 for string_segment in format.split(','):
797 if '-' in string_segment:
798 start, end = string_segment.split('-')
799 for item in range(int(start), int(end) + 1):
800 yield int(item)
801 else:
802 yield int(string_segment)
803 playlistitems = iter_playlistitems(playlistitems_str)
804
805 ie_entries = ie_result['entries']
806 if isinstance(ie_entries, list):
807 n_all_entries = len(ie_entries)
808 if playlistitems:
809 entries = [
810 ie_entries[i - 1] for i in playlistitems
811 if -n_all_entries <= i - 1 < n_all_entries]
812 else:
813 entries = ie_entries[playliststart:playlistend]
814 n_entries = len(entries)
815 self.to_screen(
816 '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
817 (ie_result['extractor'], playlist, n_all_entries, n_entries))
818 elif isinstance(ie_entries, PagedList):
819 if playlistitems:
820 entries = []
821 for item in playlistitems:
822 entries.extend(ie_entries.getslice(
823 item - 1, item
824 ))
825 else:
826 entries = ie_entries.getslice(
827 playliststart, playlistend)
828 n_entries = len(entries)
829 self.to_screen(
830 '[%s] playlist %s: Downloading %d videos' %
831 (ie_result['extractor'], playlist, n_entries))
832 else: # iterable
833 if playlistitems:
834 entry_list = list(ie_entries)
835 entries = [entry_list[i - 1] for i in playlistitems]
836 else:
837 entries = list(itertools.islice(
838 ie_entries, playliststart, playlistend))
839 n_entries = len(entries)
840 self.to_screen(
841 '[%s] playlist %s: Downloading %d videos' %
842 (ie_result['extractor'], playlist, n_entries))
843
844 if self.params.get('playlistreverse', False):
845 entries = entries[::-1]
846
847 if self.params.get('playlistrandom', False):
848 random.shuffle(entries)
849
850 for i, entry in enumerate(entries, 1):
851 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
852 extra = {
853 'n_entries': n_entries,
854 'playlist': playlist,
855 'playlist_id': ie_result.get('id'),
856 'playlist_title': ie_result.get('title'),
857 'playlist_index': i + playliststart,
858 'extractor': ie_result['extractor'],
859 'webpage_url': ie_result['webpage_url'],
860 'webpage_url_basename': url_basename(ie_result['webpage_url']),
861 'extractor_key': ie_result['extractor_key'],
862 }
863
864 reason = self._match_entry(entry, incomplete=True)
865 if reason is not None:
866 self.to_screen('[download] ' + reason)
867 continue
868
869 entry_result = self.process_ie_result(entry,
870 download=download,
871 extra_info=extra)
872 playlist_results.append(entry_result)
873 ie_result['entries'] = playlist_results
874 self.to_screen('[download] Finished downloading playlist: %s' % playlist)
875 return ie_result
876 elif result_type == 'compat_list':
877 self.report_warning(
878 'Extractor %s returned a compat_list result. '
879 'It needs to be updated.' % ie_result.get('extractor'))
880
881 def _fixup(r):
882 self.add_extra_info(
883 r,
884 {
885 'extractor': ie_result['extractor'],
886 'webpage_url': ie_result['webpage_url'],
887 'webpage_url_basename': url_basename(ie_result['webpage_url']),
888 'extractor_key': ie_result['extractor_key'],
889 }
890 )
891 return r
892 ie_result['entries'] = [
893 self.process_ie_result(_fixup(r), download, extra_info)
894 for r in ie_result['entries']
895 ]
896 return ie_result
897 else:
898 raise Exception('Invalid result type: %s' % result_type)
899
900 def _build_format_filter(self, filter_spec):
901 " Returns a function to filter the formats according to the filter_spec "
902
903 OPERATORS = {
904 '<': operator.lt,
905 '<=': operator.le,
906 '>': operator.gt,
907 '>=': operator.ge,
908 '=': operator.eq,
909 '!=': operator.ne,
910 }
911 operator_rex = re.compile(r'''(?x)\s*
912 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
913 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
914 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
915 $
916 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
917 m = operator_rex.search(filter_spec)
918 if m:
919 try:
920 comparison_value = int(m.group('value'))
921 except ValueError:
922 comparison_value = parse_filesize(m.group('value'))
923 if comparison_value is None:
924 comparison_value = parse_filesize(m.group('value') + 'B')
925 if comparison_value is None:
926 raise ValueError(
927 'Invalid value %r in format specification %r' % (
928 m.group('value'), filter_spec))
929 op = OPERATORS[m.group('op')]
930
931 if not m:
932 STR_OPERATORS = {
933 '=': operator.eq,
934 '!=': operator.ne,
935 '^=': lambda attr, value: attr.startswith(value),
936 '$=': lambda attr, value: attr.endswith(value),
937 '*=': lambda attr, value: value in attr,
938 }
939 str_operator_rex = re.compile(r'''(?x)
940 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
941 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
942 \s*(?P<value>[a-zA-Z0-9._-]+)
943 \s*$
944 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
945 m = str_operator_rex.search(filter_spec)
946 if m:
947 comparison_value = m.group('value')
948 op = STR_OPERATORS[m.group('op')]
949
950 if not m:
951 raise ValueError('Invalid filter specification %r' % filter_spec)
952
953 def _filter(f):
954 actual_value = f.get(m.group('key'))
955 if actual_value is None:
956 return m.group('none_inclusive')
957 return op(actual_value, comparison_value)
958 return _filter
959
960 def build_format_selector(self, format_spec):
961 def syntax_error(note, start):
962 message = (
963 'Invalid format specification: '
964 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
965 return SyntaxError(message)
966
967 PICKFIRST = 'PICKFIRST'
968 MERGE = 'MERGE'
969 SINGLE = 'SINGLE'
970 GROUP = 'GROUP'
971 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
972
973 def _parse_filter(tokens):
974 filter_parts = []
975 for type, string, start, _, _ in tokens:
976 if type == tokenize.OP and string == ']':
977 return ''.join(filter_parts)
978 else:
979 filter_parts.append(string)
980
981 def _remove_unused_ops(tokens):
982 # Remove operators that we don't use and join them with the surrounding strings
983 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
984 ALLOWED_OPS = ('/', '+', ',', '(', ')')
985 last_string, last_start, last_end, last_line = None, None, None, None
986 for type, string, start, end, line in tokens:
987 if type == tokenize.OP and string == '[':
988 if last_string:
989 yield tokenize.NAME, last_string, last_start, last_end, last_line
990 last_string = None
991 yield type, string, start, end, line
992 # everything inside brackets will be handled by _parse_filter
993 for type, string, start, end, line in tokens:
994 yield type, string, start, end, line
995 if type == tokenize.OP and string == ']':
996 break
997 elif type == tokenize.OP and string in ALLOWED_OPS:
998 if last_string:
999 yield tokenize.NAME, last_string, last_start, last_end, last_line
1000 last_string = None
1001 yield type, string, start, end, line
1002 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1003 if not last_string:
1004 last_string = string
1005 last_start = start
1006 last_end = end
1007 else:
1008 last_string += string
1009 if last_string:
1010 yield tokenize.NAME, last_string, last_start, last_end, last_line
1011
1012 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1013 selectors = []
1014 current_selector = None
1015 for type, string, start, _, _ in tokens:
1016 # ENCODING is only defined in python 3.x
1017 if type == getattr(tokenize, 'ENCODING', None):
1018 continue
1019 elif type in [tokenize.NAME, tokenize.NUMBER]:
1020 current_selector = FormatSelector(SINGLE, string, [])
1021 elif type == tokenize.OP:
1022 if string == ')':
1023 if not inside_group:
1024 # ')' will be handled by the parentheses group
1025 tokens.restore_last_token()
1026 break
1027 elif inside_merge and string in ['/', ',']:
1028 tokens.restore_last_token()
1029 break
1030 elif inside_choice and string == ',':
1031 tokens.restore_last_token()
1032 break
1033 elif string == ',':
1034 if not current_selector:
1035 raise syntax_error('"," must follow a format selector', start)
1036 selectors.append(current_selector)
1037 current_selector = None
1038 elif string == '/':
1039 if not current_selector:
1040 raise syntax_error('"/" must follow a format selector', start)
1041 first_choice = current_selector
1042 second_choice = _parse_format_selection(tokens, inside_choice=True)
1043 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1044 elif string == '[':
1045 if not current_selector:
1046 current_selector = FormatSelector(SINGLE, 'best', [])
1047 format_filter = _parse_filter(tokens)
1048 current_selector.filters.append(format_filter)
1049 elif string == '(':
1050 if current_selector:
1051 raise syntax_error('Unexpected "("', start)
1052 group = _parse_format_selection(tokens, inside_group=True)
1053 current_selector = FormatSelector(GROUP, group, [])
1054 elif string == '+':
1055 video_selector = current_selector
1056 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1057 if not video_selector or not audio_selector:
1058 raise syntax_error('"+" must be between two format selectors', start)
1059 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1060 else:
1061 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1062 elif type == tokenize.ENDMARKER:
1063 break
1064 if current_selector:
1065 selectors.append(current_selector)
1066 return selectors
1067
1068 def _build_selector_function(selector):
1069 if isinstance(selector, list):
1070 fs = [_build_selector_function(s) for s in selector]
1071
1072 def selector_function(ctx):
1073 for f in fs:
1074 for format in f(ctx):
1075 yield format
1076 return selector_function
1077 elif selector.type == GROUP:
1078 selector_function = _build_selector_function(selector.selector)
1079 elif selector.type == PICKFIRST:
1080 fs = [_build_selector_function(s) for s in selector.selector]
1081
1082 def selector_function(ctx):
1083 for f in fs:
1084 picked_formats = list(f(ctx))
1085 if picked_formats:
1086 return picked_formats
1087 return []
1088 elif selector.type == SINGLE:
1089 format_spec = selector.selector
1090
1091 def selector_function(ctx):
1092 formats = list(ctx['formats'])
1093 if not formats:
1094 return
1095 if format_spec == 'all':
1096 for f in formats:
1097 yield f
1098 elif format_spec in ['best', 'worst', None]:
1099 format_idx = 0 if format_spec == 'worst' else -1
1100 audiovideo_formats = [
1101 f for f in formats
1102 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1103 if audiovideo_formats:
1104 yield audiovideo_formats[format_idx]
1105 # for extractors with incomplete formats (audio only (soundcloud)
1106 # or video only (imgur)) we will fallback to best/worst
1107 # {video,audio}-only format
1108 elif ctx['incomplete_formats']:
1109 yield formats[format_idx]
1110 elif format_spec == 'bestaudio':
1111 audio_formats = [
1112 f for f in formats
1113 if f.get('vcodec') == 'none']
1114 if audio_formats:
1115 yield audio_formats[-1]
1116 elif format_spec == 'worstaudio':
1117 audio_formats = [
1118 f for f in formats
1119 if f.get('vcodec') == 'none']
1120 if audio_formats:
1121 yield audio_formats[0]
1122 elif format_spec == 'bestvideo':
1123 video_formats = [
1124 f for f in formats
1125 if f.get('acodec') == 'none']
1126 if video_formats:
1127 yield video_formats[-1]
1128 elif format_spec == 'worstvideo':
1129 video_formats = [
1130 f for f in formats
1131 if f.get('acodec') == 'none']
1132 if video_formats:
1133 yield video_formats[0]
1134 else:
1135 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1136 if format_spec in extensions:
1137 filter_f = lambda f: f['ext'] == format_spec
1138 else:
1139 filter_f = lambda f: f['format_id'] == format_spec
1140 matches = list(filter(filter_f, formats))
1141 if matches:
1142 yield matches[-1]
1143 elif selector.type == MERGE:
1144 def _merge(formats_info):
1145 format_1, format_2 = [f['format_id'] for f in formats_info]
1146 # The first format must contain the video and the
1147 # second the audio
1148 if formats_info[0].get('vcodec') == 'none':
1149 self.report_error('The first format must '
1150 'contain the video, try using '
1151 '"-f %s+%s"' % (format_2, format_1))
1152 return
1153 # Formats must be opposite (video+audio)
1154 if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1155 self.report_error(
1156 'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1157 % (format_1, format_2))
1158 return
1159 output_ext = (
1160 formats_info[0]['ext']
1161 if self.params.get('merge_output_format') is None
1162 else self.params['merge_output_format'])
1163 return {
1164 'requested_formats': formats_info,
1165 'format': '%s+%s' % (formats_info[0].get('format'),
1166 formats_info[1].get('format')),
1167 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1168 formats_info[1].get('format_id')),
1169 'width': formats_info[0].get('width'),
1170 'height': formats_info[0].get('height'),
1171 'resolution': formats_info[0].get('resolution'),
1172 'fps': formats_info[0].get('fps'),
1173 'vcodec': formats_info[0].get('vcodec'),
1174 'vbr': formats_info[0].get('vbr'),
1175 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1176 'acodec': formats_info[1].get('acodec'),
1177 'abr': formats_info[1].get('abr'),
1178 'ext': output_ext,
1179 }
1180 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1181
1182 def selector_function(ctx):
1183 for pair in itertools.product(
1184 video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1185 yield _merge(pair)
1186
1187 filters = [self._build_format_filter(f) for f in selector.filters]
1188
1189 def final_selector(ctx):
1190 ctx_copy = copy.deepcopy(ctx)
1191 for _filter in filters:
1192 ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1193 return selector_function(ctx_copy)
1194 return final_selector
1195
1196 stream = io.BytesIO(format_spec.encode('utf-8'))
1197 try:
1198 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1199 except tokenize.TokenError:
1200 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1201
1202 class TokenIterator(object):
1203 def __init__(self, tokens):
1204 self.tokens = tokens
1205 self.counter = 0
1206
1207 def __iter__(self):
1208 return self
1209
1210 def __next__(self):
1211 if self.counter >= len(self.tokens):
1212 raise StopIteration()
1213 value = self.tokens[self.counter]
1214 self.counter += 1
1215 return value
1216
1217 next = __next__
1218
1219 def restore_last_token(self):
1220 self.counter -= 1
1221
1222 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1223 return _build_selector_function(parsed_selector)
1224
1225 def _calc_headers(self, info_dict):
1226 res = std_headers.copy()
1227
1228 add_headers = info_dict.get('http_headers')
1229 if add_headers:
1230 res.update(add_headers)
1231
1232 cookies = self._calc_cookies(info_dict)
1233 if cookies:
1234 res['Cookie'] = cookies
1235
1236 return res
1237
1238 def _calc_cookies(self, info_dict):
1239 pr = sanitized_Request(info_dict['url'])
1240 self.cookiejar.add_cookie_header(pr)
1241 return pr.get_header('Cookie')
1242
1243 def process_video_result(self, info_dict, download=True):
1244 assert info_dict.get('_type', 'video') == 'video'
1245
1246 if 'id' not in info_dict:
1247 raise ExtractorError('Missing "id" field in extractor result')
1248 if 'title' not in info_dict:
1249 raise ExtractorError('Missing "title" field in extractor result')
1250
1251 if not isinstance(info_dict['id'], compat_str):
1252 self.report_warning('"id" field is not a string - forcing string conversion')
1253 info_dict['id'] = compat_str(info_dict['id'])
1254
1255 if 'playlist' not in info_dict:
1256 # It isn't part of a playlist
1257 info_dict['playlist'] = None
1258 info_dict['playlist_index'] = None
1259
1260 thumbnails = info_dict.get('thumbnails')
1261 if thumbnails is None:
1262 thumbnail = info_dict.get('thumbnail')
1263 if thumbnail:
1264 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1265 if thumbnails:
1266 thumbnails.sort(key=lambda t: (
1267 t.get('preference') if t.get('preference') is not None else -1,
1268 t.get('width') if t.get('width') is not None else -1,
1269 t.get('height') if t.get('height') is not None else -1,
1270 t.get('id') if t.get('id') is not None else '', t.get('url')))
1271 for i, t in enumerate(thumbnails):
1272 t['url'] = sanitize_url(t['url'])
1273 if t.get('width') and t.get('height'):
1274 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1275 if t.get('id') is None:
1276 t['id'] = '%d' % i
1277
1278 if self.params.get('list_thumbnails'):
1279 self.list_thumbnails(info_dict)
1280 return
1281
1282 thumbnail = info_dict.get('thumbnail')
1283 if thumbnail:
1284 info_dict['thumbnail'] = sanitize_url(thumbnail)
1285 elif thumbnails:
1286 info_dict['thumbnail'] = thumbnails[-1]['url']
1287
1288 if 'display_id' not in info_dict and 'id' in info_dict:
1289 info_dict['display_id'] = info_dict['id']
1290
1291 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1292 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1293 # see http://bugs.python.org/issue1646728)
1294 try:
1295 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1296 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1297 except (ValueError, OverflowError, OSError):
1298 pass
1299
1300 # Auto generate title fields corresponding to the *_number fields when missing
1301 # in order to always have clean titles. This is very common for TV series.
1302 for field in ('chapter', 'season', 'episode'):
1303 if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1304 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1305
1306 subtitles = info_dict.get('subtitles')
1307 if subtitles:
1308 for _, subtitle in subtitles.items():
1309 for subtitle_format in subtitle:
1310 if subtitle_format.get('url'):
1311 subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1312 if subtitle_format.get('ext') is None:
1313 subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1314
1315 if self.params.get('listsubtitles', False):
1316 if 'automatic_captions' in info_dict:
1317 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1318 self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1319 return
1320 info_dict['requested_subtitles'] = self.process_subtitles(
1321 info_dict['id'], subtitles,
1322 info_dict.get('automatic_captions'))
1323
1324 # We now pick which formats have to be downloaded
1325 if info_dict.get('formats') is None:
1326 # There's only one format available
1327 formats = [info_dict]
1328 else:
1329 formats = info_dict['formats']
1330
1331 if not formats:
1332 raise ExtractorError('No video formats found!')
1333
1334 formats_dict = {}
1335
1336 # We check that all the formats have the format and format_id fields
1337 for i, format in enumerate(formats):
1338 if 'url' not in format:
1339 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1340
1341 format['url'] = sanitize_url(format['url'])
1342
1343 if format.get('format_id') is None:
1344 format['format_id'] = compat_str(i)
1345 else:
1346 # Sanitize format_id from characters used in format selector expression
1347 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1348 format_id = format['format_id']
1349 if format_id not in formats_dict:
1350 formats_dict[format_id] = []
1351 formats_dict[format_id].append(format)
1352
1353 # Make sure all formats have unique format_id
1354 for format_id, ambiguous_formats in formats_dict.items():
1355 if len(ambiguous_formats) > 1:
1356 for i, format in enumerate(ambiguous_formats):
1357 format['format_id'] = '%s-%d' % (format_id, i)
1358
1359 for i, format in enumerate(formats):
1360 if format.get('format') is None:
1361 format['format'] = '{id} - {res}{note}'.format(
1362 id=format['format_id'],
1363 res=self.format_resolution(format),
1364 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1365 )
1366 # Automatically determine file extension if missing
1367 if format.get('ext') is None:
1368 format['ext'] = determine_ext(format['url']).lower()
1369 # Automatically determine protocol if missing (useful for format
1370 # selection purposes)
1371 if format.get('protocol') is None:
1372 format['protocol'] = determine_protocol(format)
1373 # Add HTTP headers, so that external programs can use them from the
1374 # json output
1375 full_format_info = info_dict.copy()
1376 full_format_info.update(format)
1377 format['http_headers'] = self._calc_headers(full_format_info)
1378
1379 # TODO Central sorting goes here
1380
1381 if formats[0] is not info_dict:
1382 # only set the 'formats' fields if the original info_dict list them
1383 # otherwise we end up with a circular reference, the first (and unique)
1384 # element in the 'formats' field in info_dict is info_dict itself,
1385 # which can't be exported to json
1386 info_dict['formats'] = formats
1387 if self.params.get('listformats'):
1388 self.list_formats(info_dict)
1389 return
1390
1391 req_format = self.params.get('format')
1392 if req_format is None:
1393 req_format_list = []
1394 if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1395 not info_dict.get('is_live')):
1396 merger = FFmpegMergerPP(self)
1397 if merger.available and merger.can_merge():
1398 req_format_list.append('bestvideo+bestaudio')
1399 req_format_list.append('best')
1400 req_format = '/'.join(req_format_list)
1401 format_selector = self.build_format_selector(req_format)
1402
1403 # While in format selection we may need to have an access to the original
1404 # format set in order to calculate some metrics or do some processing.
1405 # For now we need to be able to guess whether original formats provided
1406 # by extractor are incomplete or not (i.e. whether extractor provides only
1407 # video-only or audio-only formats) for proper formats selection for
1408 # extractors with such incomplete formats (see
1409 # https://github.com/rg3/youtube-dl/pull/5556).
1410 # Since formats may be filtered during format selection and may not match
1411 # the original formats the results may be incorrect. Thus original formats
1412 # or pre-calculated metrics should be passed to format selection routines
1413 # as well.
1414 # We will pass a context object containing all necessary additional data
1415 # instead of just formats.
1416 # This fixes incorrect format selection issue (see
1417 # https://github.com/rg3/youtube-dl/issues/10083).
1418 incomplete_formats = (
1419 # All formats are video-only or
1420 all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1421 # all formats are audio-only
1422 all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1423
1424 ctx = {
1425 'formats': formats,
1426 'incomplete_formats': incomplete_formats,
1427 }
1428
1429 formats_to_download = list(format_selector(ctx))
1430 if not formats_to_download:
1431 raise ExtractorError('requested format not available',
1432 expected=True)
1433
1434 if download:
1435 if len(formats_to_download) > 1:
1436 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1437 for format in formats_to_download:
1438 new_info = dict(info_dict)
1439 new_info.update(format)
1440 self.process_info(new_info)
1441 # We update the info dict with the best quality format (backwards compatibility)
1442 info_dict.update(formats_to_download[-1])
1443 return info_dict
1444
1445 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1446 """Select the requested subtitles and their format"""
1447 available_subs = {}
1448 if normal_subtitles and self.params.get('writesubtitles'):
1449 available_subs.update(normal_subtitles)
1450 if automatic_captions and self.params.get('writeautomaticsub'):
1451 for lang, cap_info in automatic_captions.items():
1452 if lang not in available_subs:
1453 available_subs[lang] = cap_info
1454
1455 if (not self.params.get('writesubtitles') and not
1456 self.params.get('writeautomaticsub') or not
1457 available_subs):
1458 return None
1459
1460 if self.params.get('allsubtitles', False):
1461 requested_langs = available_subs.keys()
1462 else:
1463 if self.params.get('subtitleslangs', False):
1464 requested_langs = self.params.get('subtitleslangs')
1465 elif 'en' in available_subs:
1466 requested_langs = ['en']
1467 else:
1468 requested_langs = [list(available_subs.keys())[0]]
1469
1470 formats_query = self.params.get('subtitlesformat', 'best')
1471 formats_preference = formats_query.split('/') if formats_query else []
1472 subs = {}
1473 for lang in requested_langs:
1474 formats = available_subs.get(lang)
1475 if formats is None:
1476 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1477 continue
1478 for ext in formats_preference:
1479 if ext == 'best':
1480 f = formats[-1]
1481 break
1482 matches = list(filter(lambda f: f['ext'] == ext, formats))
1483 if matches:
1484 f = matches[-1]
1485 break
1486 else:
1487 f = formats[-1]
1488 self.report_warning(
1489 'No subtitle format found matching "%s" for language %s, '
1490 'using %s' % (formats_query, lang, f['ext']))
1491 subs[lang] = f
1492 return subs
1493
1494 def process_info(self, info_dict):
1495 """Process a single resolved IE result."""
1496
1497 assert info_dict.get('_type', 'video') == 'video'
1498
1499 max_downloads = self.params.get('max_downloads')
1500 if max_downloads is not None:
1501 if self._num_downloads >= int(max_downloads):
1502 raise MaxDownloadsReached()
1503
1504 info_dict['fulltitle'] = info_dict['title']
1505 if len(info_dict['title']) > 200:
1506 info_dict['title'] = info_dict['title'][:197] + '...'
1507
1508 if 'format' not in info_dict:
1509 info_dict['format'] = info_dict['ext']
1510
1511 reason = self._match_entry(info_dict, incomplete=False)
1512 if reason is not None:
1513 self.to_screen('[download] ' + reason)
1514 return
1515
1516 self._num_downloads += 1
1517
1518 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1519
1520 # Forced printings
1521 if self.params.get('forcetitle', False):
1522 self.to_stdout(info_dict['fulltitle'])
1523 if self.params.get('forceid', False):
1524 self.to_stdout(info_dict['id'])
1525 if self.params.get('forceurl', False):
1526 if info_dict.get('requested_formats') is not None:
1527 for f in info_dict['requested_formats']:
1528 self.to_stdout(f['url'] + f.get('play_path', ''))
1529 else:
1530 # For RTMP URLs, also include the playpath
1531 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1532 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1533 self.to_stdout(info_dict['thumbnail'])
1534 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1535 self.to_stdout(info_dict['description'])
1536 if self.params.get('forcefilename', False) and filename is not None:
1537 self.to_stdout(filename)
1538 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1539 self.to_stdout(formatSeconds(info_dict['duration']))
1540 if self.params.get('forceformat', False):
1541 self.to_stdout(info_dict['format'])
1542 if self.params.get('forcejson', False):
1543 self.to_stdout(json.dumps(info_dict))
1544
1545 # Do nothing else if in simulate mode
1546 if self.params.get('simulate', False):
1547 return
1548
1549 if filename is None:
1550 return
1551
1552 try:
1553 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1554 if dn and not os.path.exists(dn):
1555 os.makedirs(dn)
1556 except (OSError, IOError) as err:
1557 self.report_error('unable to create directory ' + error_to_compat_str(err))
1558 return
1559
1560 if self.params.get('writedescription', False):
1561 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1562 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1563 self.to_screen('[info] Video description is already present')
1564 elif info_dict.get('description') is None:
1565 self.report_warning('There\'s no description to write.')
1566 else:
1567 try:
1568 self.to_screen('[info] Writing video description to: ' + descfn)
1569 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1570 descfile.write(info_dict['description'])
1571 except (OSError, IOError):
1572 self.report_error('Cannot write description file ' + descfn)
1573 return
1574
1575 if self.params.get('writeannotations', False):
1576 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1577 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1578 self.to_screen('[info] Video annotations are already present')
1579 else:
1580 try:
1581 self.to_screen('[info] Writing video annotations to: ' + annofn)
1582 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1583 annofile.write(info_dict['annotations'])
1584 except (KeyError, TypeError):
1585 self.report_warning('There are no annotations to write.')
1586 except (OSError, IOError):
1587 self.report_error('Cannot write annotations file: ' + annofn)
1588 return
1589
1590 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1591 self.params.get('writeautomaticsub')])
1592
1593 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1594 # subtitles download errors are already managed as troubles in relevant IE
1595 # that way it will silently go on when used with unsupporting IE
1596 subtitles = info_dict['requested_subtitles']
1597 ie = self.get_info_extractor(info_dict['extractor_key'])
1598 for sub_lang, sub_info in subtitles.items():
1599 sub_format = sub_info['ext']
1600 if sub_info.get('data') is not None:
1601 sub_data = sub_info['data']
1602 else:
1603 try:
1604 sub_data = ie._download_webpage(
1605 sub_info['url'], info_dict['id'], note=False)
1606 except ExtractorError as err:
1607 self.report_warning('Unable to download subtitle for "%s": %s' %
1608 (sub_lang, error_to_compat_str(err.cause)))
1609 continue
1610 try:
1611 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1612 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1613 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1614 else:
1615 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1616 # Use newline='' to prevent conversion of newline characters
1617 # See https://github.com/rg3/youtube-dl/issues/10268
1618 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1619 subfile.write(sub_data)
1620 except (OSError, IOError):
1621 self.report_error('Cannot write subtitles file ' + sub_filename)
1622 return
1623
1624 if self.params.get('writeinfojson', False):
1625 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1626 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1627 self.to_screen('[info] Video description metadata is already present')
1628 else:
1629 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1630 try:
1631 write_json_file(self.filter_requested_info(info_dict), infofn)
1632 except (OSError, IOError):
1633 self.report_error('Cannot write metadata to JSON file ' + infofn)
1634 return
1635
1636 self._write_thumbnails(info_dict, filename)
1637
1638 if not self.params.get('skip_download', False):
1639 try:
1640 def dl(name, info):
1641 fd = get_suitable_downloader(info, self.params)(self, self.params)
1642 for ph in self._progress_hooks:
1643 fd.add_progress_hook(ph)
1644 if self.params.get('verbose'):
1645 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1646 return fd.download(name, info)
1647
1648 if info_dict.get('requested_formats') is not None:
1649 downloaded = []
1650 success = True
1651 merger = FFmpegMergerPP(self)
1652 if not merger.available:
1653 postprocessors = []
1654 self.report_warning('You have requested multiple '
1655 'formats but ffmpeg or avconv are not installed.'
1656 ' The formats won\'t be merged.')
1657 else:
1658 postprocessors = [merger]
1659
1660 def compatible_formats(formats):
1661 video, audio = formats
1662 # Check extension
1663 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1664 if video_ext and audio_ext:
1665 COMPATIBLE_EXTS = (
1666 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1667 ('webm')
1668 )
1669 for exts in COMPATIBLE_EXTS:
1670 if video_ext in exts and audio_ext in exts:
1671 return True
1672 # TODO: Check acodec/vcodec
1673 return False
1674
1675 filename_real_ext = os.path.splitext(filename)[1][1:]
1676 filename_wo_ext = (
1677 os.path.splitext(filename)[0]
1678 if filename_real_ext == info_dict['ext']
1679 else filename)
1680 requested_formats = info_dict['requested_formats']
1681 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1682 info_dict['ext'] = 'mkv'
1683 self.report_warning(
1684 'Requested formats are incompatible for merge and will be merged into mkv.')
1685 # Ensure filename always has a correct extension for successful merge
1686 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1687 if os.path.exists(encodeFilename(filename)):
1688 self.to_screen(
1689 '[download] %s has already been downloaded and '
1690 'merged' % filename)
1691 else:
1692 for f in requested_formats:
1693 new_info = dict(info_dict)
1694 new_info.update(f)
1695 fname = self.prepare_filename(new_info)
1696 fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1697 downloaded.append(fname)
1698 partial_success = dl(fname, new_info)
1699 success = success and partial_success
1700 info_dict['__postprocessors'] = postprocessors
1701 info_dict['__files_to_merge'] = downloaded
1702 else:
1703 # Just a single file
1704 success = dl(filename, info_dict)
1705 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1706 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1707 return
1708 except (OSError, IOError) as err:
1709 raise UnavailableVideoError(err)
1710 except (ContentTooShortError, ) as err:
1711 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1712 return
1713
1714 if success and filename != '-':
1715 # Fixup content
1716 fixup_policy = self.params.get('fixup')
1717 if fixup_policy is None:
1718 fixup_policy = 'detect_or_warn'
1719
1720 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1721
1722 stretched_ratio = info_dict.get('stretched_ratio')
1723 if stretched_ratio is not None and stretched_ratio != 1:
1724 if fixup_policy == 'warn':
1725 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1726 info_dict['id'], stretched_ratio))
1727 elif fixup_policy == 'detect_or_warn':
1728 stretched_pp = FFmpegFixupStretchedPP(self)
1729 if stretched_pp.available:
1730 info_dict.setdefault('__postprocessors', [])
1731 info_dict['__postprocessors'].append(stretched_pp)
1732 else:
1733 self.report_warning(
1734 '%s: Non-uniform pixel ratio (%s). %s'
1735 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1736 else:
1737 assert fixup_policy in ('ignore', 'never')
1738
1739 if (info_dict.get('requested_formats') is None and
1740 info_dict.get('container') == 'm4a_dash'):
1741 if fixup_policy == 'warn':
1742 self.report_warning(
1743 '%s: writing DASH m4a. '
1744 'Only some players support this container.'
1745 % info_dict['id'])
1746 elif fixup_policy == 'detect_or_warn':
1747 fixup_pp = FFmpegFixupM4aPP(self)
1748 if fixup_pp.available:
1749 info_dict.setdefault('__postprocessors', [])
1750 info_dict['__postprocessors'].append(fixup_pp)
1751 else:
1752 self.report_warning(
1753 '%s: writing DASH m4a. '
1754 'Only some players support this container. %s'
1755 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1756 else:
1757 assert fixup_policy in ('ignore', 'never')
1758
1759 if (info_dict.get('protocol') == 'm3u8_native' or
1760 info_dict.get('protocol') == 'm3u8' and
1761 self.params.get('hls_prefer_native')):
1762 if fixup_policy == 'warn':
1763 self.report_warning('%s: malformated aac bitstream.' % (
1764 info_dict['id']))
1765 elif fixup_policy == 'detect_or_warn':
1766 fixup_pp = FFmpegFixupM3u8PP(self)
1767 if fixup_pp.available:
1768 info_dict.setdefault('__postprocessors', [])
1769 info_dict['__postprocessors'].append(fixup_pp)
1770 else:
1771 self.report_warning(
1772 '%s: malformated aac bitstream. %s'
1773 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1774 else:
1775 assert fixup_policy in ('ignore', 'never')
1776
1777 try:
1778 self.post_process(filename, info_dict)
1779 except (PostProcessingError) as err:
1780 self.report_error('postprocessing: %s' % str(err))
1781 return
1782 self.record_download_archive(info_dict)
1783
1784 def download(self, url_list):
1785 """Download a given list of URLs."""
1786 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1787 if (len(url_list) > 1 and
1788 '%' not in outtmpl and
1789 self.params.get('max_downloads') != 1):
1790 raise SameFileError(outtmpl)
1791
1792 for url in url_list:
1793 try:
1794 # It also downloads the videos
1795 res = self.extract_info(
1796 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1797 except UnavailableVideoError:
1798 self.report_error('unable to download video')
1799 except MaxDownloadsReached:
1800 self.to_screen('[info] Maximum number of downloaded files reached.')
1801 raise
1802 else:
1803 if self.params.get('dump_single_json', False):
1804 self.to_stdout(json.dumps(res))
1805
1806 return self._download_retcode
1807
1808 def download_with_info_file(self, info_filename):
1809 with contextlib.closing(fileinput.FileInput(
1810 [info_filename], mode='r',
1811 openhook=fileinput.hook_encoded('utf-8'))) as f:
1812 # FileInput doesn't have a read method, we can't call json.load
1813 info = self.filter_requested_info(json.loads('\n'.join(f)))
1814 try:
1815 self.process_ie_result(info, download=True)
1816 except DownloadError:
1817 webpage_url = info.get('webpage_url')
1818 if webpage_url is not None:
1819 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1820 return self.download([webpage_url])
1821 else:
1822 raise
1823 return self._download_retcode
1824
1825 @staticmethod
1826 def filter_requested_info(info_dict):
1827 return dict(
1828 (k, v) for k, v in info_dict.items()
1829 if k not in ['requested_formats', 'requested_subtitles'])
1830
1831 def post_process(self, filename, ie_info):
1832 """Run all the postprocessors on the given file."""
1833 info = dict(ie_info)
1834 info['filepath'] = filename
1835 pps_chain = []
1836 if ie_info.get('__postprocessors') is not None:
1837 pps_chain.extend(ie_info['__postprocessors'])
1838 pps_chain.extend(self._pps)
1839 for pp in pps_chain:
1840 files_to_delete = []
1841 try:
1842 files_to_delete, info = pp.run(info)
1843 except PostProcessingError as e:
1844 self.report_error(e.msg)
1845 if files_to_delete and not self.params.get('keepvideo', False):
1846 for old_filename in files_to_delete:
1847 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1848 try:
1849 os.remove(encodeFilename(old_filename))
1850 except (IOError, OSError):
1851 self.report_warning('Unable to remove downloaded original file')
1852
1853 def _make_archive_id(self, info_dict):
1854 # Future-proof against any change in case
1855 # and backwards compatibility with prior versions
1856 extractor = info_dict.get('extractor_key')
1857 if extractor is None:
1858 if 'id' in info_dict:
1859 extractor = info_dict.get('ie_key') # key in a playlist
1860 if extractor is None:
1861 return None # Incomplete video information
1862 return extractor.lower() + ' ' + info_dict['id']
1863
1864 def in_download_archive(self, info_dict):
1865 fn = self.params.get('download_archive')
1866 if fn is None:
1867 return False
1868
1869 vid_id = self._make_archive_id(info_dict)
1870 if vid_id is None:
1871 return False # Incomplete video information
1872
1873 try:
1874 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1875 for line in archive_file:
1876 if line.strip() == vid_id:
1877 return True
1878 except IOError as ioe:
1879 if ioe.errno != errno.ENOENT:
1880 raise
1881 return False
1882
1883 def record_download_archive(self, info_dict):
1884 fn = self.params.get('download_archive')
1885 if fn is None:
1886 return
1887 vid_id = self._make_archive_id(info_dict)
1888 assert vid_id
1889 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1890 archive_file.write(vid_id + '\n')
1891
1892 @staticmethod
1893 def format_resolution(format, default='unknown'):
1894 if format.get('vcodec') == 'none':
1895 return 'audio only'
1896 if format.get('resolution') is not None:
1897 return format['resolution']
1898 if format.get('height') is not None:
1899 if format.get('width') is not None:
1900 res = '%sx%s' % (format['width'], format['height'])
1901 else:
1902 res = '%sp' % format['height']
1903 elif format.get('width') is not None:
1904 res = '%dx?' % format['width']
1905 else:
1906 res = default
1907 return res
1908
1909 def _format_note(self, fdict):
1910 res = ''
1911 if fdict.get('ext') in ['f4f', 'f4m']:
1912 res += '(unsupported) '
1913 if fdict.get('language'):
1914 if res:
1915 res += ' '
1916 res += '[%s] ' % fdict['language']
1917 if fdict.get('format_note') is not None:
1918 res += fdict['format_note'] + ' '
1919 if fdict.get('tbr') is not None:
1920 res += '%4dk ' % fdict['tbr']
1921 if fdict.get('container') is not None:
1922 if res:
1923 res += ', '
1924 res += '%s container' % fdict['container']
1925 if (fdict.get('vcodec') is not None and
1926 fdict.get('vcodec') != 'none'):
1927 if res:
1928 res += ', '
1929 res += fdict['vcodec']
1930 if fdict.get('vbr') is not None:
1931 res += '@'
1932 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1933 res += 'video@'
1934 if fdict.get('vbr') is not None:
1935 res += '%4dk' % fdict['vbr']
1936 if fdict.get('fps') is not None:
1937 if res:
1938 res += ', '
1939 res += '%sfps' % fdict['fps']
1940 if fdict.get('acodec') is not None:
1941 if res:
1942 res += ', '
1943 if fdict['acodec'] == 'none':
1944 res += 'video only'
1945 else:
1946 res += '%-5s' % fdict['acodec']
1947 elif fdict.get('abr') is not None:
1948 if res:
1949 res += ', '
1950 res += 'audio'
1951 if fdict.get('abr') is not None:
1952 res += '@%3dk' % fdict['abr']
1953 if fdict.get('asr') is not None:
1954 res += ' (%5dHz)' % fdict['asr']
1955 if fdict.get('filesize') is not None:
1956 if res:
1957 res += ', '
1958 res += format_bytes(fdict['filesize'])
1959 elif fdict.get('filesize_approx') is not None:
1960 if res:
1961 res += ', '
1962 res += '~' + format_bytes(fdict['filesize_approx'])
1963 return res
1964
1965 def list_formats(self, info_dict):
1966 formats = info_dict.get('formats', [info_dict])
1967 table = [
1968 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1969 for f in formats
1970 if f.get('preference') is None or f['preference'] >= -1000]
1971 if len(formats) > 1:
1972 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1973
1974 header_line = ['format code', 'extension', 'resolution', 'note']
1975 self.to_screen(
1976 '[info] Available formats for %s:\n%s' %
1977 (info_dict['id'], render_table(header_line, table)))
1978
1979 def list_thumbnails(self, info_dict):
1980 thumbnails = info_dict.get('thumbnails')
1981 if not thumbnails:
1982 self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
1983 return
1984
1985 self.to_screen(
1986 '[info] Thumbnails for %s:' % info_dict['id'])
1987 self.to_screen(render_table(
1988 ['ID', 'width', 'height', 'URL'],
1989 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1990
1991 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1992 if not subtitles:
1993 self.to_screen('%s has no %s' % (video_id, name))
1994 return
1995 self.to_screen(
1996 'Available %s for %s:' % (name, video_id))
1997 self.to_screen(render_table(
1998 ['Language', 'formats'],
1999 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2000 for lang, formats in subtitles.items()]))
2001
2002 def urlopen(self, req):
2003 """ Start an HTTP download """
2004 if isinstance(req, compat_basestring):
2005 req = sanitized_Request(req)
2006 return self._opener.open(req, timeout=self._socket_timeout)
2007
2008 def print_debug_header(self):
2009 if not self.params.get('verbose'):
2010 return
2011
2012 if type('') is not compat_str:
2013 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2014 self.report_warning(
2015 'Your Python is broken! Update to a newer and supported version')
2016
2017 stdout_encoding = getattr(
2018 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2019 encoding_str = (
2020 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2021 locale.getpreferredencoding(),
2022 sys.getfilesystemencoding(),
2023 stdout_encoding,
2024 self.get_encoding()))
2025 write_string(encoding_str, encoding=None)
2026
2027 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2028 if _LAZY_LOADER:
2029 self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2030 try:
2031 sp = subprocess.Popen(
2032 ['git', 'rev-parse', '--short', 'HEAD'],
2033 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2034 cwd=os.path.dirname(os.path.abspath(__file__)))
2035 out, err = sp.communicate()
2036 out = out.decode().strip()
2037 if re.match('[0-9a-f]+', out):
2038 self._write_string('[debug] Git HEAD: ' + out + '\n')
2039 except Exception:
2040 try:
2041 sys.exc_clear()
2042 except Exception:
2043 pass
2044 self._write_string('[debug] Python version %s - %s\n' % (
2045 platform.python_version(), platform_name()))
2046
2047 exe_versions = FFmpegPostProcessor.get_versions(self)
2048 exe_versions['rtmpdump'] = rtmpdump_version()
2049 exe_str = ', '.join(
2050 '%s %s' % (exe, v)
2051 for exe, v in sorted(exe_versions.items())
2052 if v
2053 )
2054 if not exe_str:
2055 exe_str = 'none'
2056 self._write_string('[debug] exe versions: %s\n' % exe_str)
2057
2058 proxy_map = {}
2059 for handler in self._opener.handlers:
2060 if hasattr(handler, 'proxies'):
2061 proxy_map.update(handler.proxies)
2062 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2063
2064 if self.params.get('call_home', False):
2065 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2066 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2067 latest_version = self.urlopen(
2068 'https://yt-dl.org/latest/version').read().decode('utf-8')
2069 if version_tuple(latest_version) > version_tuple(__version__):
2070 self.report_warning(
2071 'You are using an outdated version (newest version: %s)! '
2072 'See https://yt-dl.org/update if you need help updating.' %
2073 latest_version)
2074
2075 def _setup_opener(self):
2076 timeout_val = self.params.get('socket_timeout')
2077 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2078
2079 opts_cookiefile = self.params.get('cookiefile')
2080 opts_proxy = self.params.get('proxy')
2081
2082 if opts_cookiefile is None:
2083 self.cookiejar = compat_cookiejar.CookieJar()
2084 else:
2085 opts_cookiefile = compat_expanduser(opts_cookiefile)
2086 self.cookiejar = compat_cookiejar.MozillaCookieJar(
2087 opts_cookiefile)
2088 if os.access(opts_cookiefile, os.R_OK):
2089 self.cookiejar.load()
2090
2091 cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2092 if opts_proxy is not None:
2093 if opts_proxy == '':
2094 proxies = {}
2095 else:
2096 proxies = {'http': opts_proxy, 'https': opts_proxy}
2097 else:
2098 proxies = compat_urllib_request.getproxies()
2099 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2100 if 'http' in proxies and 'https' not in proxies:
2101 proxies['https'] = proxies['http']
2102 proxy_handler = PerRequestProxyHandler(proxies)
2103
2104 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2105 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2106 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2107 data_handler = compat_urllib_request_DataHandler()
2108
2109 # When passing our own FileHandler instance, build_opener won't add the
2110 # default FileHandler and allows us to disable the file protocol, which
2111 # can be used for malicious purposes (see
2112 # https://github.com/rg3/youtube-dl/issues/8227)
2113 file_handler = compat_urllib_request.FileHandler()
2114
2115 def file_open(*args, **kwargs):
2116 raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2117 file_handler.file_open = file_open
2118
2119 opener = compat_urllib_request.build_opener(
2120 proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2121
2122 # Delete the default user-agent header, which would otherwise apply in
2123 # cases where our custom HTTP handler doesn't come into play
2124 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2125 opener.addheaders = []
2126 self._opener = opener
2127
2128 def encode(self, s):
2129 if isinstance(s, bytes):
2130 return s # Already encoded
2131
2132 try:
2133 return s.encode(self.get_encoding())
2134 except UnicodeEncodeError as err:
2135 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2136 raise
2137
2138 def get_encoding(self):
2139 encoding = self.params.get('encoding')
2140 if encoding is None:
2141 encoding = preferredencoding()
2142 return encoding
2143
2144 def _write_thumbnails(self, info_dict, filename):
2145 if self.params.get('writethumbnail', False):
2146 thumbnails = info_dict.get('thumbnails')
2147 if thumbnails:
2148 thumbnails = [thumbnails[-1]]
2149 elif self.params.get('write_all_thumbnails', False):
2150 thumbnails = info_dict.get('thumbnails')
2151 else:
2152 return
2153
2154 if not thumbnails:
2155 # No thumbnails present, so return immediately
2156 return
2157
2158 for t in thumbnails:
2159 thumb_ext = determine_ext(t['url'], 'jpg')
2160 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2161 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2162 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2163
2164 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2165 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2166 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2167 else:
2168 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2169 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2170 try:
2171 uf = self.urlopen(t['url'])
2172 with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2173 shutil.copyfileobj(uf, thumbf)
2174 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2175 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2176 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2177 self.report_warning('Unable to download thumbnail "%s": %s' %
2178 (t['url'], error_to_compat_str(err)))