]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py
Update changelog.
[youtubedl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import absolute_import, unicode_literals
5
6 import collections
7 import errno
8 import io
9 import json
10 import os
11 import platform
12 import re
13 import shutil
14 import subprocess
15 import socket
16 import sys
17 import time
18 import traceback
19
20 if os.name == 'nt':
21 import ctypes
22
23 from .utils import (
24 compat_cookiejar,
25 compat_http_client,
26 compat_str,
27 compat_urllib_error,
28 compat_urllib_request,
29 ContentTooShortError,
30 date_from_str,
31 DateRange,
32 determine_ext,
33 DownloadError,
34 encodeFilename,
35 ExtractorError,
36 format_bytes,
37 formatSeconds,
38 get_term_width,
39 locked_file,
40 make_HTTPS_handler,
41 MaxDownloadsReached,
42 PagedList,
43 PostProcessingError,
44 platform_name,
45 preferredencoding,
46 SameFileError,
47 sanitize_filename,
48 subtitles_filename,
49 takewhile_inclusive,
50 UnavailableVideoError,
51 url_basename,
52 write_json_file,
53 write_string,
54 YoutubeDLHandler,
55 prepend_extension,
56 )
57 from .extractor import get_info_extractor, gen_extractors
58 from .downloader import get_suitable_downloader
59 from .postprocessor import FFmpegMergerPP
60 from .version import __version__
61
62
63 class YoutubeDL(object):
64 """YoutubeDL class.
65
66 YoutubeDL objects are the ones responsible of downloading the
67 actual video file and writing it to disk if the user has requested
68 it, among some other tasks. In most cases there should be one per
69 program. As, given a video URL, the downloader doesn't know how to
70 extract all the needed information, task that InfoExtractors do, it
71 has to pass the URL to one of them.
72
73 For this, YoutubeDL objects have a method that allows
74 InfoExtractors to be registered in a given order. When it is passed
75 a URL, the YoutubeDL object handles it to the first InfoExtractor it
76 finds that reports being able to handle it. The InfoExtractor extracts
77 all the information about the video or videos the URL refers to, and
78 YoutubeDL process the extracted information, possibly using a File
79 Downloader to download the video.
80
81 YoutubeDL objects accept a lot of parameters. In order not to saturate
82 the object constructor with arguments, it receives a dictionary of
83 options instead. These options are available through the params
84 attribute for the InfoExtractors to use. The YoutubeDL also
85 registers itself as the downloader in charge for the InfoExtractors
86 that are added to it, so this is a "mutual registration".
87
88 Available options:
89
90 username: Username for authentication purposes.
91 password: Password for authentication purposes.
92 videopassword: Password for acces a video.
93 usenetrc: Use netrc for authentication instead.
94 verbose: Print additional info to stdout.
95 quiet: Do not print messages to stdout.
96 forceurl: Force printing final URL.
97 forcetitle: Force printing title.
98 forceid: Force printing ID.
99 forcethumbnail: Force printing thumbnail URL.
100 forcedescription: Force printing description.
101 forcefilename: Force printing final filename.
102 forceduration: Force printing duration.
103 forcejson: Force printing info_dict as JSON.
104 simulate: Do not download the video files.
105 format: Video format code.
106 format_limit: Highest quality format to try.
107 outtmpl: Template for output names.
108 restrictfilenames: Do not allow "&" and spaces in file names
109 ignoreerrors: Do not stop on download errors.
110 nooverwrites: Prevent overwriting files.
111 playliststart: Playlist item to start at.
112 playlistend: Playlist item to end at.
113 matchtitle: Download only matching titles.
114 rejecttitle: Reject downloads for matching titles.
115 logger: Log messages to a logging.Logger instance.
116 logtostderr: Log messages to stderr instead of stdout.
117 writedescription: Write the video description to a .description file
118 writeinfojson: Write the video description to a .info.json file
119 writeannotations: Write the video annotations to a .annotations.xml file
120 writethumbnail: Write the thumbnail image to a file
121 writesubtitles: Write the video subtitles to a file
122 writeautomaticsub: Write the automatic subtitles to a file
123 allsubtitles: Downloads all the subtitles of the video
124 (requires writesubtitles or writeautomaticsub)
125 listsubtitles: Lists all available subtitles for the video
126 subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
127 subtitleslangs: List of languages of the subtitles to download
128 keepvideo: Keep the video file after post-processing
129 daterange: A DateRange object, download only if the upload_date is in the range.
130 skip_download: Skip the actual download of the video file
131 cachedir: Location of the cache files in the filesystem.
132 None to disable filesystem cache.
133 noplaylist: Download single video instead of a playlist if in doubt.
134 age_limit: An integer representing the user's age in years.
135 Unsuitable videos for the given age are skipped.
136 min_views: An integer representing the minimum view count the video
137 must have in order to not be skipped.
138 Videos without view count information are always
139 downloaded. None for no limit.
140 max_views: An integer representing the maximum view count.
141 Videos that are more popular than that are not
142 downloaded.
143 Videos without view count information are always
144 downloaded. None for no limit.
145 download_archive: File name of a file where all downloads are recorded.
146 Videos already present in the file are not downloaded
147 again.
148 cookiefile: File name where cookies should be read from and dumped to.
149 nocheckcertificate:Do not verify SSL certificates
150 proxy: URL of the proxy server to use
151 socket_timeout: Time to wait for unresponsive hosts, in seconds
152 bidi_workaround: Work around buggy terminals without bidirectional text
153 support, using fridibi
154 debug_printtraffic:Print out sent and received HTTP traffic
155 include_ads: Download ads as well
156 default_search: Prepend this string if an input url is not valid.
157 'auto' for elaborate guessing
158
159 The following parameters are not used by YoutubeDL itself, they are used by
160 the FileDownloader:
161 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
162 noresizebuffer, retries, continuedl, noprogress, consoletitle
163
164 The following options are used by the post processors:
165 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
166 otherwise prefer avconv.
167 """
168
169 params = None
170 _ies = []
171 _pps = []
172 _download_retcode = None
173 _num_downloads = None
174 _screen_file = None
175
176 def __init__(self, params=None):
177 """Create a FileDownloader object with the given options."""
178 if params is None:
179 params = {}
180 self._ies = []
181 self._ies_instances = {}
182 self._pps = []
183 self._progress_hooks = []
184 self._download_retcode = 0
185 self._num_downloads = 0
186 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
187 self._err_file = sys.stderr
188 self.params = params
189
190 if params.get('bidi_workaround', False):
191 try:
192 import pty
193 master, slave = pty.openpty()
194 width = get_term_width()
195 if width is None:
196 width_args = []
197 else:
198 width_args = ['-w', str(width)]
199 sp_kwargs = dict(
200 stdin=subprocess.PIPE,
201 stdout=slave,
202 stderr=self._err_file)
203 try:
204 self._output_process = subprocess.Popen(
205 ['bidiv'] + width_args, **sp_kwargs
206 )
207 except OSError:
208 self._output_process = subprocess.Popen(
209 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
210 self._output_channel = os.fdopen(master, 'rb')
211 except OSError as ose:
212 if ose.errno == 2:
213 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
214 else:
215 raise
216
217 if (sys.version_info >= (3,) and sys.platform != 'win32' and
218 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
219 and not params['restrictfilenames']):
220 # On Python 3, the Unicode filesystem API will throw errors (#1474)
221 self.report_warning(
222 'Assuming --restrict-filenames since file system encoding '
223 'cannot encode all charactes. '
224 'Set the LC_ALL environment variable to fix this.')
225 self.params['restrictfilenames'] = True
226
227 if '%(stitle)s' in self.params.get('outtmpl', ''):
228 self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
229
230 self._setup_opener()
231
232 def add_info_extractor(self, ie):
233 """Add an InfoExtractor object to the end of the list."""
234 self._ies.append(ie)
235 self._ies_instances[ie.ie_key()] = ie
236 ie.set_downloader(self)
237
238 def get_info_extractor(self, ie_key):
239 """
240 Get an instance of an IE with name ie_key, it will try to get one from
241 the _ies list, if there's no instance it will create a new one and add
242 it to the extractor list.
243 """
244 ie = self._ies_instances.get(ie_key)
245 if ie is None:
246 ie = get_info_extractor(ie_key)()
247 self.add_info_extractor(ie)
248 return ie
249
250 def add_default_info_extractors(self):
251 """
252 Add the InfoExtractors returned by gen_extractors to the end of the list
253 """
254 for ie in gen_extractors():
255 self.add_info_extractor(ie)
256
257 def add_post_processor(self, pp):
258 """Add a PostProcessor object to the end of the chain."""
259 self._pps.append(pp)
260 pp.set_downloader(self)
261
262 def add_progress_hook(self, ph):
263 """Add the progress hook (currently only for the file downloader)"""
264 self._progress_hooks.append(ph)
265
266 def _bidi_workaround(self, message):
267 if not hasattr(self, '_output_channel'):
268 return message
269
270 assert hasattr(self, '_output_process')
271 assert type(message) == type('')
272 line_count = message.count('\n') + 1
273 self._output_process.stdin.write((message + '\n').encode('utf-8'))
274 self._output_process.stdin.flush()
275 res = ''.join(self._output_channel.readline().decode('utf-8')
276 for _ in range(line_count))
277 return res[:-len('\n')]
278
279 def to_screen(self, message, skip_eol=False):
280 """Print message to stdout if not in quiet mode."""
281 return self.to_stdout(message, skip_eol, check_quiet=True)
282
283 def to_stdout(self, message, skip_eol=False, check_quiet=False):
284 """Print message to stdout if not in quiet mode."""
285 if self.params.get('logger'):
286 self.params['logger'].debug(message)
287 elif not check_quiet or not self.params.get('quiet', False):
288 message = self._bidi_workaround(message)
289 terminator = ['\n', ''][skip_eol]
290 output = message + terminator
291
292 write_string(output, self._screen_file)
293
294 def to_stderr(self, message):
295 """Print message to stderr."""
296 assert type(message) == type('')
297 if self.params.get('logger'):
298 self.params['logger'].error(message)
299 else:
300 message = self._bidi_workaround(message)
301 output = message + '\n'
302 write_string(output, self._err_file)
303
304 def to_console_title(self, message):
305 if not self.params.get('consoletitle', False):
306 return
307 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
308 # c_wchar_p() might not be necessary if `message` is
309 # already of type unicode()
310 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
311 elif 'TERM' in os.environ:
312 write_string('\033]0;%s\007' % message, self._screen_file)
313
314 def save_console_title(self):
315 if not self.params.get('consoletitle', False):
316 return
317 if 'TERM' in os.environ:
318 # Save the title on stack
319 write_string('\033[22;0t', self._screen_file)
320
321 def restore_console_title(self):
322 if not self.params.get('consoletitle', False):
323 return
324 if 'TERM' in os.environ:
325 # Restore the title from stack
326 write_string('\033[23;0t', self._screen_file)
327
328 def __enter__(self):
329 self.save_console_title()
330 return self
331
332 def __exit__(self, *args):
333 self.restore_console_title()
334
335 if self.params.get('cookiefile') is not None:
336 self.cookiejar.save()
337
338 def trouble(self, message=None, tb=None):
339 """Determine action to take when a download problem appears.
340
341 Depending on if the downloader has been configured to ignore
342 download errors or not, this method may throw an exception or
343 not when errors are found, after printing the message.
344
345 tb, if given, is additional traceback information.
346 """
347 if message is not None:
348 self.to_stderr(message)
349 if self.params.get('verbose'):
350 if tb is None:
351 if sys.exc_info()[0]: # if .trouble has been called from an except block
352 tb = ''
353 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
354 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
355 tb += compat_str(traceback.format_exc())
356 else:
357 tb_data = traceback.format_list(traceback.extract_stack())
358 tb = ''.join(tb_data)
359 self.to_stderr(tb)
360 if not self.params.get('ignoreerrors', False):
361 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
362 exc_info = sys.exc_info()[1].exc_info
363 else:
364 exc_info = sys.exc_info()
365 raise DownloadError(message, exc_info)
366 self._download_retcode = 1
367
368 def report_warning(self, message):
369 '''
370 Print the message to stderr, it will be prefixed with 'WARNING:'
371 If stderr is a tty file the 'WARNING:' will be colored
372 '''
373 if self._err_file.isatty() and os.name != 'nt':
374 _msg_header = '\033[0;33mWARNING:\033[0m'
375 else:
376 _msg_header = 'WARNING:'
377 warning_message = '%s %s' % (_msg_header, message)
378 self.to_stderr(warning_message)
379
380 def report_error(self, message, tb=None):
381 '''
382 Do the same as trouble, but prefixes the message with 'ERROR:', colored
383 in red if stderr is a tty file.
384 '''
385 if self._err_file.isatty() and os.name != 'nt':
386 _msg_header = '\033[0;31mERROR:\033[0m'
387 else:
388 _msg_header = 'ERROR:'
389 error_message = '%s %s' % (_msg_header, message)
390 self.trouble(error_message, tb)
391
392 def report_file_already_downloaded(self, file_name):
393 """Report file has already been fully downloaded."""
394 try:
395 self.to_screen('[download] %s has already been downloaded' % file_name)
396 except UnicodeEncodeError:
397 self.to_screen('[download] The file has already been downloaded')
398
399 def prepare_filename(self, info_dict):
400 """Generate the output filename."""
401 try:
402 template_dict = dict(info_dict)
403
404 template_dict['epoch'] = int(time.time())
405 autonumber_size = self.params.get('autonumber_size')
406 if autonumber_size is None:
407 autonumber_size = 5
408 autonumber_templ = '%0' + str(autonumber_size) + 'd'
409 template_dict['autonumber'] = autonumber_templ % self._num_downloads
410 if template_dict.get('playlist_index') is not None:
411 template_dict['playlist_index'] = '%05d' % template_dict['playlist_index']
412
413 sanitize = lambda k, v: sanitize_filename(
414 compat_str(v),
415 restricted=self.params.get('restrictfilenames'),
416 is_id=(k == 'id'))
417 template_dict = dict((k, sanitize(k, v))
418 for k, v in template_dict.items()
419 if v is not None)
420 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
421
422 tmpl = os.path.expanduser(self.params['outtmpl'])
423 filename = tmpl % template_dict
424 return filename
425 except ValueError as err:
426 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
427 return None
428
429 def _match_entry(self, info_dict):
430 """ Returns None iff the file should be downloaded """
431
432 video_title = info_dict.get('title', info_dict.get('id', 'video'))
433 if 'title' in info_dict:
434 # This can happen when we're just evaluating the playlist
435 title = info_dict['title']
436 matchtitle = self.params.get('matchtitle', False)
437 if matchtitle:
438 if not re.search(matchtitle, title, re.IGNORECASE):
439 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
440 rejecttitle = self.params.get('rejecttitle', False)
441 if rejecttitle:
442 if re.search(rejecttitle, title, re.IGNORECASE):
443 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
444 date = info_dict.get('upload_date', None)
445 if date is not None:
446 dateRange = self.params.get('daterange', DateRange())
447 if date not in dateRange:
448 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
449 view_count = info_dict.get('view_count', None)
450 if view_count is not None:
451 min_views = self.params.get('min_views')
452 if min_views is not None and view_count < min_views:
453 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
454 max_views = self.params.get('max_views')
455 if max_views is not None and view_count > max_views:
456 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
457 age_limit = self.params.get('age_limit')
458 if age_limit is not None:
459 if age_limit < info_dict.get('age_limit', 0):
460 return 'Skipping "' + title + '" because it is age restricted'
461 if self.in_download_archive(info_dict):
462 return '%s has already been recorded in archive' % video_title
463 return None
464
465 @staticmethod
466 def add_extra_info(info_dict, extra_info):
467 '''Set the keys from extra_info in info dict if they are missing'''
468 for key, value in extra_info.items():
469 info_dict.setdefault(key, value)
470
471 def extract_info(self, url, download=True, ie_key=None, extra_info={},
472 process=True):
473 '''
474 Returns a list with a dictionary for each video we find.
475 If 'download', also downloads the videos.
476 extra_info is a dict containing the extra values to add to each result
477 '''
478
479 if ie_key:
480 ies = [self.get_info_extractor(ie_key)]
481 else:
482 ies = self._ies
483
484 for ie in ies:
485 if not ie.suitable(url):
486 continue
487
488 if not ie.working():
489 self.report_warning('The program functionality for this site has been marked as broken, '
490 'and will probably not work.')
491
492 try:
493 ie_result = ie.extract(url)
494 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
495 break
496 if isinstance(ie_result, list):
497 # Backwards compatibility: old IE result format
498 ie_result = {
499 '_type': 'compat_list',
500 'entries': ie_result,
501 }
502 self.add_extra_info(ie_result,
503 {
504 'extractor': ie.IE_NAME,
505 'webpage_url': url,
506 'webpage_url_basename': url_basename(url),
507 'extractor_key': ie.ie_key(),
508 })
509 if process:
510 return self.process_ie_result(ie_result, download, extra_info)
511 else:
512 return ie_result
513 except ExtractorError as de: # An error we somewhat expected
514 self.report_error(compat_str(de), de.format_traceback())
515 break
516 except MaxDownloadsReached:
517 raise
518 except Exception as e:
519 if self.params.get('ignoreerrors', False):
520 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
521 break
522 else:
523 raise
524 else:
525 self.report_error('no suitable InfoExtractor: %s' % url)
526
527 def process_ie_result(self, ie_result, download=True, extra_info={}):
528 """
529 Take the result of the ie(may be modified) and resolve all unresolved
530 references (URLs, playlist items).
531
532 It will also download the videos if 'download'.
533 Returns the resolved ie_result.
534 """
535
536 result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
537 if result_type == 'video':
538 self.add_extra_info(ie_result, extra_info)
539 return self.process_video_result(ie_result, download=download)
540 elif result_type == 'url':
541 # We have to add extra_info to the results because it may be
542 # contained in a playlist
543 return self.extract_info(ie_result['url'],
544 download,
545 ie_key=ie_result.get('ie_key'),
546 extra_info=extra_info)
547 elif result_type == 'url_transparent':
548 # Use the information from the embedding page
549 info = self.extract_info(
550 ie_result['url'], ie_key=ie_result.get('ie_key'),
551 extra_info=extra_info, download=False, process=False)
552
553 def make_result(embedded_info):
554 new_result = ie_result.copy()
555 for f in ('_type', 'url', 'ext', 'player_url', 'formats',
556 'entries', 'ie_key', 'duration',
557 'subtitles', 'annotations', 'format',
558 'thumbnail', 'thumbnails'):
559 if f in new_result:
560 del new_result[f]
561 if f in embedded_info:
562 new_result[f] = embedded_info[f]
563 return new_result
564 new_result = make_result(info)
565
566 assert new_result.get('_type') != 'url_transparent'
567 if new_result.get('_type') == 'compat_list':
568 new_result['entries'] = [
569 make_result(e) for e in new_result['entries']]
570
571 return self.process_ie_result(
572 new_result, download=download, extra_info=extra_info)
573 elif result_type == 'playlist':
574 # We process each entry in the playlist
575 playlist = ie_result.get('title', None) or ie_result.get('id', None)
576 self.to_screen('[download] Downloading playlist: %s' % playlist)
577
578 playlist_results = []
579
580 playliststart = self.params.get('playliststart', 1) - 1
581 playlistend = self.params.get('playlistend', None)
582 # For backwards compatibility, interpret -1 as whole list
583 if playlistend == -1:
584 playlistend = None
585
586 if isinstance(ie_result['entries'], list):
587 n_all_entries = len(ie_result['entries'])
588 entries = ie_result['entries'][playliststart:playlistend]
589 n_entries = len(entries)
590 self.to_screen(
591 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
592 (ie_result['extractor'], playlist, n_all_entries, n_entries))
593 else:
594 assert isinstance(ie_result['entries'], PagedList)
595 entries = ie_result['entries'].getslice(
596 playliststart, playlistend)
597 n_entries = len(entries)
598 self.to_screen(
599 "[%s] playlist %s: Downloading %d videos" %
600 (ie_result['extractor'], playlist, n_entries))
601
602 for i, entry in enumerate(entries, 1):
603 self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
604 extra = {
605 'playlist': playlist,
606 'playlist_index': i + playliststart,
607 'extractor': ie_result['extractor'],
608 'webpage_url': ie_result['webpage_url'],
609 'webpage_url_basename': url_basename(ie_result['webpage_url']),
610 'extractor_key': ie_result['extractor_key'],
611 }
612
613 reason = self._match_entry(entry)
614 if reason is not None:
615 self.to_screen('[download] ' + reason)
616 continue
617
618 entry_result = self.process_ie_result(entry,
619 download=download,
620 extra_info=extra)
621 playlist_results.append(entry_result)
622 ie_result['entries'] = playlist_results
623 return ie_result
624 elif result_type == 'compat_list':
625 def _fixup(r):
626 self.add_extra_info(r,
627 {
628 'extractor': ie_result['extractor'],
629 'webpage_url': ie_result['webpage_url'],
630 'webpage_url_basename': url_basename(ie_result['webpage_url']),
631 'extractor_key': ie_result['extractor_key'],
632 })
633 return r
634 ie_result['entries'] = [
635 self.process_ie_result(_fixup(r), download, extra_info)
636 for r in ie_result['entries']
637 ]
638 return ie_result
639 else:
640 raise Exception('Invalid result type: %s' % result_type)
641
642 def select_format(self, format_spec, available_formats):
643 if format_spec == 'best' or format_spec is None:
644 return available_formats[-1]
645 elif format_spec == 'worst':
646 return available_formats[0]
647 elif format_spec == 'bestaudio':
648 audio_formats = [
649 f for f in available_formats
650 if f.get('vcodec') == 'none']
651 if audio_formats:
652 return audio_formats[-1]
653 elif format_spec == 'worstaudio':
654 audio_formats = [
655 f for f in available_formats
656 if f.get('vcodec') == 'none']
657 if audio_formats:
658 return audio_formats[0]
659 else:
660 extensions = ['mp4', 'flv', 'webm', '3gp']
661 if format_spec in extensions:
662 filter_f = lambda f: f['ext'] == format_spec
663 else:
664 filter_f = lambda f: f['format_id'] == format_spec
665 matches = list(filter(filter_f, available_formats))
666 if matches:
667 return matches[-1]
668 return None
669
670 def process_video_result(self, info_dict, download=True):
671 assert info_dict.get('_type', 'video') == 'video'
672
673 if 'playlist' not in info_dict:
674 # It isn't part of a playlist
675 info_dict['playlist'] = None
676 info_dict['playlist_index'] = None
677
678 # This extractors handle format selection themselves
679 if info_dict['extractor'] in ['Youku']:
680 if download:
681 self.process_info(info_dict)
682 return info_dict
683
684 # We now pick which formats have to be downloaded
685 if info_dict.get('formats') is None:
686 # There's only one format available
687 formats = [info_dict]
688 else:
689 formats = info_dict['formats']
690
691 # We check that all the formats have the format and format_id fields
692 for (i, format) in enumerate(formats):
693 if format.get('format_id') is None:
694 format['format_id'] = compat_str(i)
695 if format.get('format') is None:
696 format['format'] = '{id} - {res}{note}'.format(
697 id=format['format_id'],
698 res=self.format_resolution(format),
699 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
700 )
701 # Automatically determine file extension if missing
702 if 'ext' not in format:
703 format['ext'] = determine_ext(format['url'])
704
705 format_limit = self.params.get('format_limit', None)
706 if format_limit:
707 formats = list(takewhile_inclusive(
708 lambda f: f['format_id'] != format_limit, formats
709 ))
710
711 # TODO Central sorting goes here
712
713 if formats[0] is not info_dict:
714 # only set the 'formats' fields if the original info_dict list them
715 # otherwise we end up with a circular reference, the first (and unique)
716 # element in the 'formats' field in info_dict is info_dict itself,
717 # wich can't be exported to json
718 info_dict['formats'] = formats
719 if self.params.get('listformats', None):
720 self.list_formats(info_dict)
721 return
722
723 req_format = self.params.get('format')
724 if req_format is None:
725 req_format = 'best'
726 formats_to_download = []
727 # The -1 is for supporting YoutubeIE
728 if req_format in ('-1', 'all'):
729 formats_to_download = formats
730 else:
731 # We can accept formats requested in the format: 34/5/best, we pick
732 # the first that is available, starting from left
733 req_formats = req_format.split('/')
734 for rf in req_formats:
735 if re.match(r'.+?\+.+?', rf) is not None:
736 # Two formats have been requested like '137+139'
737 format_1, format_2 = rf.split('+')
738 formats_info = (self.select_format(format_1, formats),
739 self.select_format(format_2, formats))
740 if all(formats_info):
741 selected_format = {
742 'requested_formats': formats_info,
743 'format': rf,
744 'ext': formats_info[0]['ext'],
745 }
746 else:
747 selected_format = None
748 else:
749 selected_format = self.select_format(rf, formats)
750 if selected_format is not None:
751 formats_to_download = [selected_format]
752 break
753 if not formats_to_download:
754 raise ExtractorError('requested format not available',
755 expected=True)
756
757 if download:
758 if len(formats_to_download) > 1:
759 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
760 for format in formats_to_download:
761 new_info = dict(info_dict)
762 new_info.update(format)
763 self.process_info(new_info)
764 # We update the info dict with the best quality format (backwards compatibility)
765 info_dict.update(formats_to_download[-1])
766 return info_dict
767
768 def process_info(self, info_dict):
769 """Process a single resolved IE result."""
770
771 assert info_dict.get('_type', 'video') == 'video'
772
773 max_downloads = self.params.get('max_downloads')
774 if max_downloads is not None:
775 if self._num_downloads >= int(max_downloads):
776 raise MaxDownloadsReached()
777
778 info_dict['fulltitle'] = info_dict['title']
779 if len(info_dict['title']) > 200:
780 info_dict['title'] = info_dict['title'][:197] + '...'
781
782 # Keep for backwards compatibility
783 info_dict['stitle'] = info_dict['title']
784
785 if not 'format' in info_dict:
786 info_dict['format'] = info_dict['ext']
787
788 reason = self._match_entry(info_dict)
789 if reason is not None:
790 self.to_screen('[download] ' + reason)
791 return
792
793 self._num_downloads += 1
794
795 filename = self.prepare_filename(info_dict)
796
797 # Forced printings
798 if self.params.get('forcetitle', False):
799 self.to_stdout(info_dict['fulltitle'])
800 if self.params.get('forceid', False):
801 self.to_stdout(info_dict['id'])
802 if self.params.get('forceurl', False):
803 # For RTMP URLs, also include the playpath
804 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
805 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
806 self.to_stdout(info_dict['thumbnail'])
807 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
808 self.to_stdout(info_dict['description'])
809 if self.params.get('forcefilename', False) and filename is not None:
810 self.to_stdout(filename)
811 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
812 self.to_stdout(formatSeconds(info_dict['duration']))
813 if self.params.get('forceformat', False):
814 self.to_stdout(info_dict['format'])
815 if self.params.get('forcejson', False):
816 info_dict['_filename'] = filename
817 self.to_stdout(json.dumps(info_dict))
818
819 # Do nothing else if in simulate mode
820 if self.params.get('simulate', False):
821 return
822
823 if filename is None:
824 return
825
826 try:
827 dn = os.path.dirname(encodeFilename(filename))
828 if dn != '' and not os.path.exists(dn):
829 os.makedirs(dn)
830 except (OSError, IOError) as err:
831 self.report_error('unable to create directory ' + compat_str(err))
832 return
833
834 if self.params.get('writedescription', False):
835 descfn = filename + '.description'
836 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
837 self.to_screen('[info] Video description is already present')
838 else:
839 try:
840 self.to_screen('[info] Writing video description to: ' + descfn)
841 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
842 descfile.write(info_dict['description'])
843 except (KeyError, TypeError):
844 self.report_warning('There\'s no description to write.')
845 except (OSError, IOError):
846 self.report_error('Cannot write description file ' + descfn)
847 return
848
849 if self.params.get('writeannotations', False):
850 annofn = filename + '.annotations.xml'
851 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
852 self.to_screen('[info] Video annotations are already present')
853 else:
854 try:
855 self.to_screen('[info] Writing video annotations to: ' + annofn)
856 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
857 annofile.write(info_dict['annotations'])
858 except (KeyError, TypeError):
859 self.report_warning('There are no annotations to write.')
860 except (OSError, IOError):
861 self.report_error('Cannot write annotations file: ' + annofn)
862 return
863
864 subtitles_are_requested = any([self.params.get('writesubtitles', False),
865 self.params.get('writeautomaticsub')])
866
867 if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
868 # subtitles download errors are already managed as troubles in relevant IE
869 # that way it will silently go on when used with unsupporting IE
870 subtitles = info_dict['subtitles']
871 sub_format = self.params.get('subtitlesformat', 'srt')
872 for sub_lang in subtitles.keys():
873 sub = subtitles[sub_lang]
874 if sub is None:
875 continue
876 try:
877 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
878 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
879 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
880 else:
881 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
882 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
883 subfile.write(sub)
884 except (OSError, IOError):
885 self.report_error('Cannot write subtitles file ' + descfn)
886 return
887
888 if self.params.get('writeinfojson', False):
889 infofn = os.path.splitext(filename)[0] + '.info.json'
890 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
891 self.to_screen('[info] Video description metadata is already present')
892 else:
893 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
894 try:
895 write_json_file(info_dict, encodeFilename(infofn))
896 except (OSError, IOError):
897 self.report_error('Cannot write metadata to JSON file ' + infofn)
898 return
899
900 if self.params.get('writethumbnail', False):
901 if info_dict.get('thumbnail') is not None:
902 thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
903 thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
904 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
905 self.to_screen('[%s] %s: Thumbnail is already present' %
906 (info_dict['extractor'], info_dict['id']))
907 else:
908 self.to_screen('[%s] %s: Downloading thumbnail ...' %
909 (info_dict['extractor'], info_dict['id']))
910 try:
911 uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
912 with open(thumb_filename, 'wb') as thumbf:
913 shutil.copyfileobj(uf, thumbf)
914 self.to_screen('[%s] %s: Writing thumbnail to: %s' %
915 (info_dict['extractor'], info_dict['id'], thumb_filename))
916 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
917 self.report_warning('Unable to download thumbnail "%s": %s' %
918 (info_dict['thumbnail'], compat_str(err)))
919
920 if not self.params.get('skip_download', False):
921 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
922 success = True
923 else:
924 try:
925 def dl(name, info):
926 fd = get_suitable_downloader(info)(self, self.params)
927 for ph in self._progress_hooks:
928 fd.add_progress_hook(ph)
929 return fd.download(name, info)
930 if info_dict.get('requested_formats') is not None:
931 downloaded = []
932 success = True
933 merger = FFmpegMergerPP(self)
934 if not merger._get_executable():
935 postprocessors = []
936 self.report_warning('You have requested multiple '
937 'formats but ffmpeg or avconv are not installed.'
938 ' The formats won\'t be merged')
939 else:
940 postprocessors = [merger]
941 for f in info_dict['requested_formats']:
942 new_info = dict(info_dict)
943 new_info.update(f)
944 fname = self.prepare_filename(new_info)
945 fname = prepend_extension(fname, 'f%s' % f['format_id'])
946 downloaded.append(fname)
947 partial_success = dl(fname, new_info)
948 success = success and partial_success
949 info_dict['__postprocessors'] = postprocessors
950 info_dict['__files_to_merge'] = downloaded
951 else:
952 # Just a single file
953 success = dl(filename, info_dict)
954 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
955 self.report_error('unable to download video data: %s' % str(err))
956 return
957 except (OSError, IOError) as err:
958 raise UnavailableVideoError(err)
959 except (ContentTooShortError, ) as err:
960 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
961 return
962
963 if success:
964 try:
965 self.post_process(filename, info_dict)
966 except (PostProcessingError) as err:
967 self.report_error('postprocessing: %s' % str(err))
968 return
969
970 self.record_download_archive(info_dict)
971
972 def download(self, url_list):
973 """Download a given list of URLs."""
974 if (len(url_list) > 1 and
975 '%' not in self.params['outtmpl']
976 and self.params.get('max_downloads') != 1):
977 raise SameFileError(self.params['outtmpl'])
978
979 for url in url_list:
980 try:
981 #It also downloads the videos
982 self.extract_info(url)
983 except UnavailableVideoError:
984 self.report_error('unable to download video')
985 except MaxDownloadsReached:
986 self.to_screen('[info] Maximum number of downloaded files reached.')
987 raise
988
989 return self._download_retcode
990
991 def download_with_info_file(self, info_filename):
992 with io.open(info_filename, 'r', encoding='utf-8') as f:
993 info = json.load(f)
994 try:
995 self.process_ie_result(info, download=True)
996 except DownloadError:
997 webpage_url = info.get('webpage_url')
998 if webpage_url is not None:
999 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1000 return self.download([webpage_url])
1001 else:
1002 raise
1003 return self._download_retcode
1004
1005 def post_process(self, filename, ie_info):
1006 """Run all the postprocessors on the given file."""
1007 info = dict(ie_info)
1008 info['filepath'] = filename
1009 keep_video = None
1010 pps_chain = []
1011 if ie_info.get('__postprocessors') is not None:
1012 pps_chain.extend(ie_info['__postprocessors'])
1013 pps_chain.extend(self._pps)
1014 for pp in pps_chain:
1015 try:
1016 keep_video_wish, new_info = pp.run(info)
1017 if keep_video_wish is not None:
1018 if keep_video_wish:
1019 keep_video = keep_video_wish
1020 elif keep_video is None:
1021 # No clear decision yet, let IE decide
1022 keep_video = keep_video_wish
1023 except PostProcessingError as e:
1024 self.report_error(e.msg)
1025 if keep_video is False and not self.params.get('keepvideo', False):
1026 try:
1027 self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
1028 os.remove(encodeFilename(filename))
1029 except (IOError, OSError):
1030 self.report_warning('Unable to remove downloaded video file')
1031
1032 def _make_archive_id(self, info_dict):
1033 # Future-proof against any change in case
1034 # and backwards compatibility with prior versions
1035 extractor = info_dict.get('extractor_key')
1036 if extractor is None:
1037 if 'id' in info_dict:
1038 extractor = info_dict.get('ie_key') # key in a playlist
1039 if extractor is None:
1040 return None # Incomplete video information
1041 return extractor.lower() + ' ' + info_dict['id']
1042
1043 def in_download_archive(self, info_dict):
1044 fn = self.params.get('download_archive')
1045 if fn is None:
1046 return False
1047
1048 vid_id = self._make_archive_id(info_dict)
1049 if vid_id is None:
1050 return False # Incomplete video information
1051
1052 try:
1053 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1054 for line in archive_file:
1055 if line.strip() == vid_id:
1056 return True
1057 except IOError as ioe:
1058 if ioe.errno != errno.ENOENT:
1059 raise
1060 return False
1061
1062 def record_download_archive(self, info_dict):
1063 fn = self.params.get('download_archive')
1064 if fn is None:
1065 return
1066 vid_id = self._make_archive_id(info_dict)
1067 assert vid_id
1068 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1069 archive_file.write(vid_id + '\n')
1070
1071 @staticmethod
1072 def format_resolution(format, default='unknown'):
1073 if format.get('vcodec') == 'none':
1074 return 'audio only'
1075 if format.get('resolution') is not None:
1076 return format['resolution']
1077 if format.get('height') is not None:
1078 if format.get('width') is not None:
1079 res = '%sx%s' % (format['width'], format['height'])
1080 else:
1081 res = '%sp' % format['height']
1082 elif format.get('width') is not None:
1083 res = '?x%d' % format['width']
1084 else:
1085 res = default
1086 return res
1087
1088 def list_formats(self, info_dict):
1089 def format_note(fdict):
1090 res = ''
1091 if fdict.get('ext') in ['f4f', 'f4m']:
1092 res += '(unsupported) '
1093 if fdict.get('format_note') is not None:
1094 res += fdict['format_note'] + ' '
1095 if fdict.get('tbr') is not None:
1096 res += '%4dk ' % fdict['tbr']
1097 if fdict.get('container') is not None:
1098 if res:
1099 res += ', '
1100 res += '%s container' % fdict['container']
1101 if (fdict.get('vcodec') is not None and
1102 fdict.get('vcodec') != 'none'):
1103 if res:
1104 res += ', '
1105 res += fdict['vcodec']
1106 if fdict.get('vbr') is not None:
1107 res += '@'
1108 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1109 res += 'video@'
1110 if fdict.get('vbr') is not None:
1111 res += '%4dk' % fdict['vbr']
1112 if fdict.get('acodec') is not None:
1113 if res:
1114 res += ', '
1115 if fdict['acodec'] == 'none':
1116 res += 'video only'
1117 else:
1118 res += '%-5s' % fdict['acodec']
1119 elif fdict.get('abr') is not None:
1120 if res:
1121 res += ', '
1122 res += 'audio'
1123 if fdict.get('abr') is not None:
1124 res += '@%3dk' % fdict['abr']
1125 if fdict.get('asr') is not None:
1126 res += ' (%5dHz)' % fdict['asr']
1127 if fdict.get('filesize') is not None:
1128 if res:
1129 res += ', '
1130 res += format_bytes(fdict['filesize'])
1131 return res
1132
1133 def line(format, idlen=20):
1134 return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
1135 format['format_id'],
1136 format['ext'],
1137 self.format_resolution(format),
1138 format_note(format),
1139 ))
1140
1141 formats = info_dict.get('formats', [info_dict])
1142 idlen = max(len('format code'),
1143 max(len(f['format_id']) for f in formats))
1144 formats_s = [line(f, idlen) for f in formats]
1145 if len(formats) > 1:
1146 formats_s[0] += (' ' if format_note(formats[0]) else '') + '(worst)'
1147 formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)'
1148
1149 header_line = line({
1150 'format_id': 'format code', 'ext': 'extension',
1151 'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
1152 self.to_screen('[info] Available formats for %s:\n%s\n%s' %
1153 (info_dict['id'], header_line, '\n'.join(formats_s)))
1154
1155 def urlopen(self, req):
1156 """ Start an HTTP download """
1157 return self._opener.open(req)
1158
1159 def print_debug_header(self):
1160 if not self.params.get('verbose'):
1161 return
1162 write_string('[debug] youtube-dl version ' + __version__ + '\n')
1163 try:
1164 sp = subprocess.Popen(
1165 ['git', 'rev-parse', '--short', 'HEAD'],
1166 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1167 cwd=os.path.dirname(os.path.abspath(__file__)))
1168 out, err = sp.communicate()
1169 out = out.decode().strip()
1170 if re.match('[0-9a-f]+', out):
1171 write_string('[debug] Git HEAD: ' + out + '\n')
1172 except:
1173 try:
1174 sys.exc_clear()
1175 except:
1176 pass
1177 write_string('[debug] Python version %s - %s' %
1178 (platform.python_version(), platform_name()) + '\n')
1179
1180 proxy_map = {}
1181 for handler in self._opener.handlers:
1182 if hasattr(handler, 'proxies'):
1183 proxy_map.update(handler.proxies)
1184 write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1185
1186 def _setup_opener(self):
1187 timeout_val = self.params.get('socket_timeout')
1188 timeout = 600 if timeout_val is None else float(timeout_val)
1189
1190 opts_cookiefile = self.params.get('cookiefile')
1191 opts_proxy = self.params.get('proxy')
1192
1193 if opts_cookiefile is None:
1194 self.cookiejar = compat_cookiejar.CookieJar()
1195 else:
1196 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1197 opts_cookiefile)
1198 if os.access(opts_cookiefile, os.R_OK):
1199 self.cookiejar.load()
1200
1201 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1202 self.cookiejar)
1203 if opts_proxy is not None:
1204 if opts_proxy == '':
1205 proxies = {}
1206 else:
1207 proxies = {'http': opts_proxy, 'https': opts_proxy}
1208 else:
1209 proxies = compat_urllib_request.getproxies()
1210 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1211 if 'http' in proxies and 'https' not in proxies:
1212 proxies['https'] = proxies['http']
1213 proxy_handler = compat_urllib_request.ProxyHandler(proxies)
1214
1215 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1216 https_handler = make_HTTPS_handler(
1217 self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
1218 ydlh = YoutubeDLHandler(debuglevel=debuglevel)
1219 opener = compat_urllib_request.build_opener(
1220 https_handler, proxy_handler, cookie_processor, ydlh)
1221 # Delete the default user-agent header, which would otherwise apply in
1222 # cases where our custom HTTP handler doesn't come into play
1223 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1224 opener.addheaders = []
1225 self._opener = opener
1226
1227 # TODO remove this global modification
1228 compat_urllib_request.install_opener(opener)
1229 socket.setdefaulttimeout(timeout)