]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py
e2332f9b84ef2c41d6cf986553d4f999bac946a6
[youtubedl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import absolute_import
5
6 import errno
7 import io
8 import os
9 import re
10 import shutil
11 import socket
12 import sys
13 import time
14 import traceback
15
16 from .utils import *
17 from .extractor import get_info_extractor, gen_extractors
18 from .FileDownloader import FileDownloader
19
20
21 class YoutubeDL(object):
22 """YoutubeDL class.
23
24 YoutubeDL objects are the ones responsible of downloading the
25 actual video file and writing it to disk if the user has requested
26 it, among some other tasks. In most cases there should be one per
27 program. As, given a video URL, the downloader doesn't know how to
28 extract all the needed information, task that InfoExtractors do, it
29 has to pass the URL to one of them.
30
31 For this, YoutubeDL objects have a method that allows
32 InfoExtractors to be registered in a given order. When it is passed
33 a URL, the YoutubeDL object handles it to the first InfoExtractor it
34 finds that reports being able to handle it. The InfoExtractor extracts
35 all the information about the video or videos the URL refers to, and
36 YoutubeDL process the extracted information, possibly using a File
37 Downloader to download the video.
38
39 YoutubeDL objects accept a lot of parameters. In order not to saturate
40 the object constructor with arguments, it receives a dictionary of
41 options instead. These options are available through the params
42 attribute for the InfoExtractors to use. The YoutubeDL also
43 registers itself as the downloader in charge for the InfoExtractors
44 that are added to it, so this is a "mutual registration".
45
46 Available options:
47
48 username: Username for authentication purposes.
49 password: Password for authentication purposes.
50 videopassword: Password for acces a video.
51 usenetrc: Use netrc for authentication instead.
52 verbose: Print additional info to stdout.
53 quiet: Do not print messages to stdout.
54 forceurl: Force printing final URL.
55 forcetitle: Force printing title.
56 forceid: Force printing ID.
57 forcethumbnail: Force printing thumbnail URL.
58 forcedescription: Force printing description.
59 forcefilename: Force printing final filename.
60 simulate: Do not download the video files.
61 format: Video format code.
62 format_limit: Highest quality format to try.
63 outtmpl: Template for output names.
64 restrictfilenames: Do not allow "&" and spaces in file names
65 ignoreerrors: Do not stop on download errors.
66 nooverwrites: Prevent overwriting files.
67 playliststart: Playlist item to start at.
68 playlistend: Playlist item to end at.
69 matchtitle: Download only matching titles.
70 rejecttitle: Reject downloads for matching titles.
71 logtostderr: Log messages to stderr instead of stdout.
72 writedescription: Write the video description to a .description file
73 writeinfojson: Write the video description to a .info.json file
74 writeannotations: Write the video annotations to a .annotations.xml file
75 writethumbnail: Write the thumbnail image to a file
76 writesubtitles: Write the video subtitles to a file
77 writeautomaticsub: Write the automatic subtitles to a file
78 allsubtitles: Downloads all the subtitles of the video
79 (requires writesubtitles or writeautomaticsub)
80 listsubtitles: Lists all available subtitles for the video
81 subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
82 subtitleslangs: List of languages of the subtitles to download
83 keepvideo: Keep the video file after post-processing
84 daterange: A DateRange object, download only if the upload_date is in the range.
85 skip_download: Skip the actual download of the video file
86 cachedir: Location of the cache files in the filesystem.
87 None to disable filesystem cache.
88 noplaylist: Download single video instead of a playlist if in doubt.
89 age_limit: An integer representing the user's age in years.
90 Unsuitable videos for the given age are skipped.
91 downloadarchive: File name of a file where all downloads are recorded.
92 Videos already present in the file are not downloaded
93 again.
94
95 The following parameters are not used by YoutubeDL itself, they are used by
96 the FileDownloader:
97 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
98 noresizebuffer, retries, continuedl, noprogress, consoletitle
99 """
100
101 params = None
102 _ies = []
103 _pps = []
104 _download_retcode = None
105 _num_downloads = None
106 _screen_file = None
107
108 def __init__(self, params):
109 """Create a FileDownloader object with the given options."""
110 self._ies = []
111 self._ies_instances = {}
112 self._pps = []
113 self._progress_hooks = []
114 self._download_retcode = 0
115 self._num_downloads = 0
116 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
117
118 if (sys.version_info >= (3,) and sys.platform != 'win32' and
119 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
120 and not params['restrictfilenames']):
121 # On Python 3, the Unicode filesystem API will throw errors (#1474)
122 self.report_warning(
123 u'Assuming --restrict-filenames since file system encoding '
124 u'cannot encode all charactes. '
125 u'Set the LC_ALL environment variable to fix this.')
126 params['restrictfilenames'] = True
127
128 self.params = params
129 self.fd = FileDownloader(self, self.params)
130
131 if '%(stitle)s' in self.params['outtmpl']:
132 self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
133
134 def add_info_extractor(self, ie):
135 """Add an InfoExtractor object to the end of the list."""
136 self._ies.append(ie)
137 self._ies_instances[ie.ie_key()] = ie
138 ie.set_downloader(self)
139
140 def get_info_extractor(self, ie_key):
141 """
142 Get an instance of an IE with name ie_key, it will try to get one from
143 the _ies list, if there's no instance it will create a new one and add
144 it to the extractor list.
145 """
146 ie = self._ies_instances.get(ie_key)
147 if ie is None:
148 ie = get_info_extractor(ie_key)()
149 self.add_info_extractor(ie)
150 return ie
151
152 def add_default_info_extractors(self):
153 """
154 Add the InfoExtractors returned by gen_extractors to the end of the list
155 """
156 for ie in gen_extractors():
157 self.add_info_extractor(ie)
158
159 def add_post_processor(self, pp):
160 """Add a PostProcessor object to the end of the chain."""
161 self._pps.append(pp)
162 pp.set_downloader(self)
163
164 def to_screen(self, message, skip_eol=False):
165 """Print message to stdout if not in quiet mode."""
166 if not self.params.get('quiet', False):
167 terminator = [u'\n', u''][skip_eol]
168 output = message + terminator
169 write_string(output, self._screen_file)
170
171 def to_stderr(self, message):
172 """Print message to stderr."""
173 assert type(message) == type(u'')
174 output = message + u'\n'
175 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
176 output = output.encode(preferredencoding())
177 sys.stderr.write(output)
178
179 def fixed_template(self):
180 """Checks if the output template is fixed."""
181 return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
182
183 def trouble(self, message=None, tb=None):
184 """Determine action to take when a download problem appears.
185
186 Depending on if the downloader has been configured to ignore
187 download errors or not, this method may throw an exception or
188 not when errors are found, after printing the message.
189
190 tb, if given, is additional traceback information.
191 """
192 if message is not None:
193 self.to_stderr(message)
194 if self.params.get('verbose'):
195 if tb is None:
196 if sys.exc_info()[0]: # if .trouble has been called from an except block
197 tb = u''
198 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
199 tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
200 tb += compat_str(traceback.format_exc())
201 else:
202 tb_data = traceback.format_list(traceback.extract_stack())
203 tb = u''.join(tb_data)
204 self.to_stderr(tb)
205 if not self.params.get('ignoreerrors', False):
206 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
207 exc_info = sys.exc_info()[1].exc_info
208 else:
209 exc_info = sys.exc_info()
210 raise DownloadError(message, exc_info)
211 self._download_retcode = 1
212
213 def report_warning(self, message):
214 '''
215 Print the message to stderr, it will be prefixed with 'WARNING:'
216 If stderr is a tty file the 'WARNING:' will be colored
217 '''
218 if sys.stderr.isatty() and os.name != 'nt':
219 _msg_header = u'\033[0;33mWARNING:\033[0m'
220 else:
221 _msg_header = u'WARNING:'
222 warning_message = u'%s %s' % (_msg_header, message)
223 self.to_stderr(warning_message)
224
225 def report_error(self, message, tb=None):
226 '''
227 Do the same as trouble, but prefixes the message with 'ERROR:', colored
228 in red if stderr is a tty file.
229 '''
230 if sys.stderr.isatty() and os.name != 'nt':
231 _msg_header = u'\033[0;31mERROR:\033[0m'
232 else:
233 _msg_header = u'ERROR:'
234 error_message = u'%s %s' % (_msg_header, message)
235 self.trouble(error_message, tb)
236
237 def report_writedescription(self, descfn):
238 """ Report that the description file is being written """
239 self.to_screen(u'[info] Writing video description to: ' + descfn)
240
241 def report_writesubtitles(self, sub_filename):
242 """ Report that the subtitles file is being written """
243 self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
244
245 def report_writeinfojson(self, infofn):
246 """ Report that the metadata file has been written """
247 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
248
249 def report_writeannotations(self, annofn):
250 """ Report that the annotations file has been written. """
251 self.to_screen(u'[info] Writing video annotations to: ' + annofn)
252
253 def report_file_already_downloaded(self, file_name):
254 """Report file has already been fully downloaded."""
255 try:
256 self.to_screen(u'[download] %s has already been downloaded' % file_name)
257 except (UnicodeEncodeError) as err:
258 self.to_screen(u'[download] The file has already been downloaded')
259
260 def increment_downloads(self):
261 """Increment the ordinal that assigns a number to each file."""
262 self._num_downloads += 1
263
264 def prepare_filename(self, info_dict):
265 """Generate the output filename."""
266 try:
267 template_dict = dict(info_dict)
268
269 template_dict['epoch'] = int(time.time())
270 autonumber_size = self.params.get('autonumber_size')
271 if autonumber_size is None:
272 autonumber_size = 5
273 autonumber_templ = u'%0' + str(autonumber_size) + u'd'
274 template_dict['autonumber'] = autonumber_templ % self._num_downloads
275 if template_dict['playlist_index'] is not None:
276 template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
277
278 sanitize = lambda k, v: sanitize_filename(
279 u'NA' if v is None else compat_str(v),
280 restricted=self.params.get('restrictfilenames'),
281 is_id=(k == u'id'))
282 template_dict = dict((k, sanitize(k, v))
283 for k, v in template_dict.items())
284
285 tmpl = os.path.expanduser(self.params['outtmpl'])
286 filename = tmpl % template_dict
287 return filename
288 except KeyError as err:
289 self.report_error(u'Erroneous output template')
290 return None
291 except ValueError as err:
292 self.report_error(u'Error in output template: ' + str(err) + u' (encoding: ' + repr(preferredencoding()) + ')')
293 return None
294
295 def _match_entry(self, info_dict):
296 """ Returns None iff the file should be downloaded """
297
298 title = info_dict['title']
299 matchtitle = self.params.get('matchtitle', False)
300 if matchtitle:
301 if not re.search(matchtitle, title, re.IGNORECASE):
302 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
303 rejecttitle = self.params.get('rejecttitle', False)
304 if rejecttitle:
305 if re.search(rejecttitle, title, re.IGNORECASE):
306 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
307 date = info_dict.get('upload_date', None)
308 if date is not None:
309 dateRange = self.params.get('daterange', DateRange())
310 if date not in dateRange:
311 return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
312 age_limit = self.params.get('age_limit')
313 if age_limit is not None:
314 if age_limit < info_dict.get('age_limit', 0):
315 return u'Skipping "' + title + '" because it is age restricted'
316 if self.in_download_archive(info_dict):
317 return (u'%(title)s has already been recorded in archive'
318 % info_dict)
319 return None
320
321 def extract_info(self, url, download=True, ie_key=None, extra_info={}):
322 '''
323 Returns a list with a dictionary for each video we find.
324 If 'download', also downloads the videos.
325 extra_info is a dict containing the extra values to add to each result
326 '''
327
328 if ie_key:
329 ies = [self.get_info_extractor(ie_key)]
330 else:
331 ies = self._ies
332
333 for ie in ies:
334 if not ie.suitable(url):
335 continue
336
337 if not ie.working():
338 self.report_warning(u'The program functionality for this site has been marked as broken, '
339 u'and will probably not work.')
340
341 try:
342 ie_result = ie.extract(url)
343 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
344 break
345 if isinstance(ie_result, list):
346 # Backwards compatibility: old IE result format
347 for result in ie_result:
348 result.update(extra_info)
349 ie_result = {
350 '_type': 'compat_list',
351 'entries': ie_result,
352 }
353 else:
354 ie_result.update(extra_info)
355 if 'extractor' not in ie_result:
356 ie_result['extractor'] = ie.IE_NAME
357 return self.process_ie_result(ie_result, download=download)
358 except ExtractorError as de: # An error we somewhat expected
359 self.report_error(compat_str(de), de.format_traceback())
360 break
361 except Exception as e:
362 if self.params.get('ignoreerrors', False):
363 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
364 break
365 else:
366 raise
367 else:
368 self.report_error(u'no suitable InfoExtractor: %s' % url)
369
370 def process_ie_result(self, ie_result, download=True, extra_info={}):
371 """
372 Take the result of the ie(may be modified) and resolve all unresolved
373 references (URLs, playlist items).
374
375 It will also download the videos if 'download'.
376 Returns the resolved ie_result.
377 """
378
379 result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
380 if result_type == 'video':
381 ie_result.update(extra_info)
382 return self.process_video_result(ie_result)
383 elif result_type == 'url':
384 # We have to add extra_info to the results because it may be
385 # contained in a playlist
386 return self.extract_info(ie_result['url'],
387 download,
388 ie_key=ie_result.get('ie_key'),
389 extra_info=extra_info)
390 elif result_type == 'playlist':
391 # We process each entry in the playlist
392 playlist = ie_result.get('title', None) or ie_result.get('id', None)
393 self.to_screen(u'[download] Downloading playlist: %s' % playlist)
394
395 playlist_results = []
396
397 n_all_entries = len(ie_result['entries'])
398 playliststart = self.params.get('playliststart', 1) - 1
399 playlistend = self.params.get('playlistend', -1)
400
401 if playlistend == -1:
402 entries = ie_result['entries'][playliststart:]
403 else:
404 entries = ie_result['entries'][playliststart:playlistend]
405
406 n_entries = len(entries)
407
408 self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
409 (ie_result['extractor'], playlist, n_all_entries, n_entries))
410
411 for i, entry in enumerate(entries, 1):
412 self.to_screen(u'[download] Downloading video #%s of %s' % (i, n_entries))
413 extra = {
414 'playlist': playlist,
415 'playlist_index': i + playliststart,
416 }
417 if not 'extractor' in entry:
418 # We set the extractor, if it's an url it will be set then to
419 # the new extractor, but if it's already a video we must make
420 # sure it's present: see issue #877
421 entry['extractor'] = ie_result['extractor']
422 entry_result = self.process_ie_result(entry,
423 download=download,
424 extra_info=extra)
425 playlist_results.append(entry_result)
426 ie_result['entries'] = playlist_results
427 return ie_result
428 elif result_type == 'compat_list':
429 def _fixup(r):
430 r.setdefault('extractor', ie_result['extractor'])
431 return r
432 ie_result['entries'] = [
433 self.process_ie_result(_fixup(r), download=download)
434 for r in ie_result['entries']
435 ]
436 return ie_result
437 else:
438 raise Exception('Invalid result type: %s' % result_type)
439
440 def select_format(self, format_spec, available_formats):
441 if format_spec == 'best' or format_spec is None:
442 return available_formats[-1]
443 elif format_spec == 'worst':
444 return available_formats[0]
445 else:
446 extensions = [u'mp4', u'flv', u'webm', u'3gp']
447 if format_spec in extensions:
448 filter_f = lambda f: f['ext'] == format_spec
449 else:
450 filter_f = lambda f: f['format_id'] == format_spec
451 matches = list(filter(filter_f, available_formats))
452 if matches:
453 return matches[-1]
454 return None
455
456 def process_video_result(self, info_dict, download=True):
457 assert info_dict.get('_type', 'video') == 'video'
458
459 if 'playlist' not in info_dict:
460 # It isn't part of a playlist
461 info_dict['playlist'] = None
462 info_dict['playlist_index'] = None
463
464 # This extractors handle format selection themselves
465 if info_dict['extractor'] in [u'youtube', u'Youku', u'YouPorn', u'mixcloud']:
466 if download:
467 self.process_info(info_dict)
468 return info_dict
469
470 # We now pick which formats have to be downloaded
471 if info_dict.get('formats') is None:
472 # There's only one format available
473 formats = [info_dict]
474 else:
475 formats = info_dict['formats']
476
477 # We check that all the formats have the format and format_id fields
478 for (i, format) in enumerate(formats):
479 if format.get('format_id') is None:
480 format['format_id'] = compat_str(i)
481 if format.get('format') is None:
482 format['format'] = u'{id} - {res}{note}'.format(
483 id=format['format_id'],
484 res=self.format_resolution(format),
485 note=u' ({})'.format(format['format_note']) if format.get('format_note') is not None else '',
486 )
487
488 if self.params.get('listformats', None):
489 self.list_formats(info_dict)
490 return
491
492 format_limit = self.params.get('format_limit', None)
493 if format_limit:
494 formats = list(takewhile_inclusive(
495 lambda f: f['format_id'] != format_limit, formats
496 ))
497 if self.params.get('prefer_free_formats'):
498 def _free_formats_key(f):
499 try:
500 ext_ord = [u'flv', u'mp4', u'webm'].index(f['ext'])
501 except ValueError:
502 ext_ord = -1
503 # We only compare the extension if they have the same height and width
504 return (f.get('height'), f.get('width'), ext_ord)
505 formats = sorted(formats, key=_free_formats_key)
506
507 req_format = self.params.get('format', 'best')
508 if req_format is None:
509 req_format = 'best'
510 formats_to_download = []
511 # The -1 is for supporting YoutubeIE
512 if req_format in ('-1', 'all'):
513 formats_to_download = formats
514 else:
515 # We can accept formats requestd in the format: 34/5/best, we pick
516 # the first that is available, starting from left
517 req_formats = req_format.split('/')
518 for rf in req_formats:
519 selected_format = self.select_format(rf, formats)
520 if selected_format is not None:
521 formats_to_download = [selected_format]
522 break
523 if not formats_to_download:
524 raise ExtractorError(u'requested format not available')
525
526 if download:
527 if len(formats_to_download) > 1:
528 self.to_screen(u'[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
529 for format in formats_to_download:
530 new_info = dict(info_dict)
531 new_info.update(format)
532 self.process_info(new_info)
533 # We update the info dict with the best quality format (backwards compatibility)
534 info_dict.update(formats_to_download[-1])
535 return info_dict
536
537 def process_info(self, info_dict):
538 """Process a single resolved IE result."""
539
540 assert info_dict.get('_type', 'video') == 'video'
541 #We increment the download the download count here to match the previous behaviour.
542 self.increment_downloads()
543
544 info_dict['fulltitle'] = info_dict['title']
545 if len(info_dict['title']) > 200:
546 info_dict['title'] = info_dict['title'][:197] + u'...'
547
548 # Keep for backwards compatibility
549 info_dict['stitle'] = info_dict['title']
550
551 if not 'format' in info_dict:
552 info_dict['format'] = info_dict['ext']
553
554 reason = self._match_entry(info_dict)
555 if reason is not None:
556 self.to_screen(u'[download] ' + reason)
557 return
558
559 max_downloads = self.params.get('max_downloads')
560 if max_downloads is not None:
561 if self._num_downloads > int(max_downloads):
562 raise MaxDownloadsReached()
563
564 filename = self.prepare_filename(info_dict)
565
566 # Forced printings
567 if self.params.get('forcetitle', False):
568 compat_print(info_dict['title'])
569 if self.params.get('forceid', False):
570 compat_print(info_dict['id'])
571 if self.params.get('forceurl', False):
572 # For RTMP URLs, also include the playpath
573 compat_print(info_dict['url'] + info_dict.get('play_path', u''))
574 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
575 compat_print(info_dict['thumbnail'])
576 if self.params.get('forcedescription', False) and 'description' in info_dict:
577 compat_print(info_dict['description'])
578 if self.params.get('forcefilename', False) and filename is not None:
579 compat_print(filename)
580 if self.params.get('forceformat', False):
581 compat_print(info_dict['format'])
582
583 # Do nothing else if in simulate mode
584 if self.params.get('simulate', False):
585 return
586
587 if filename is None:
588 return
589
590 try:
591 dn = os.path.dirname(encodeFilename(filename))
592 if dn != '' and not os.path.exists(dn):
593 os.makedirs(dn)
594 except (OSError, IOError) as err:
595 self.report_error(u'unable to create directory ' + compat_str(err))
596 return
597
598 if self.params.get('writedescription', False):
599 try:
600 descfn = filename + u'.description'
601 self.report_writedescription(descfn)
602 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
603 descfile.write(info_dict['description'])
604 except (KeyError, TypeError):
605 self.report_warning(u'There\'s no description to write.')
606 except (OSError, IOError):
607 self.report_error(u'Cannot write description file ' + descfn)
608 return
609
610 if self.params.get('writeannotations', False):
611 try:
612 annofn = filename + u'.annotations.xml'
613 self.report_writeannotations(annofn)
614 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
615 annofile.write(info_dict['annotations'])
616 except (KeyError, TypeError):
617 self.report_warning(u'There are no annotations to write.')
618 except (OSError, IOError):
619 self.report_error(u'Cannot write annotations file: ' + annofn)
620 return
621
622 subtitles_are_requested = any([self.params.get('writesubtitles', False),
623 self.params.get('writeautomaticsub')])
624
625 if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
626 # subtitles download errors are already managed as troubles in relevant IE
627 # that way it will silently go on when used with unsupporting IE
628 subtitles = info_dict['subtitles']
629 sub_format = self.params.get('subtitlesformat')
630 for sub_lang in subtitles.keys():
631 sub = subtitles[sub_lang]
632 if sub is None:
633 continue
634 try:
635 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
636 self.report_writesubtitles(sub_filename)
637 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
638 subfile.write(sub)
639 except (OSError, IOError):
640 self.report_error(u'Cannot write subtitles file ' + descfn)
641 return
642
643 if self.params.get('writeinfojson', False):
644 infofn = filename + u'.info.json'
645 self.report_writeinfojson(infofn)
646 try:
647 json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle'])
648 write_json_file(json_info_dict, encodeFilename(infofn))
649 except (OSError, IOError):
650 self.report_error(u'Cannot write metadata to JSON file ' + infofn)
651 return
652
653 if self.params.get('writethumbnail', False):
654 if info_dict.get('thumbnail') is not None:
655 thumb_format = determine_ext(info_dict['thumbnail'], u'jpg')
656 thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
657 self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
658 (info_dict['extractor'], info_dict['id']))
659 try:
660 uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
661 with open(thumb_filename, 'wb') as thumbf:
662 shutil.copyfileobj(uf, thumbf)
663 self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
664 (info_dict['extractor'], info_dict['id'], thumb_filename))
665 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
666 self.report_warning(u'Unable to download thumbnail "%s": %s' %
667 (info_dict['thumbnail'], compat_str(err)))
668
669 if not self.params.get('skip_download', False):
670 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
671 success = True
672 else:
673 try:
674 success = self.fd._do_download(filename, info_dict)
675 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
676 self.report_error(u'unable to download video data: %s' % str(err))
677 return
678 except (OSError, IOError) as err:
679 raise UnavailableVideoError(err)
680 except (ContentTooShortError, ) as err:
681 self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
682 return
683
684 if success:
685 try:
686 self.post_process(filename, info_dict)
687 except (PostProcessingError) as err:
688 self.report_error(u'postprocessing: %s' % str(err))
689 return
690
691 self.record_download_archive(info_dict)
692
693 def download(self, url_list):
694 """Download a given list of URLs."""
695 if len(url_list) > 1 and self.fixed_template():
696 raise SameFileError(self.params['outtmpl'])
697
698 for url in url_list:
699 try:
700 #It also downloads the videos
701 videos = self.extract_info(url)
702 except UnavailableVideoError:
703 self.report_error(u'unable to download video')
704 except MaxDownloadsReached:
705 self.to_screen(u'[info] Maximum number of downloaded files reached.')
706 raise
707
708 return self._download_retcode
709
710 def post_process(self, filename, ie_info):
711 """Run all the postprocessors on the given file."""
712 info = dict(ie_info)
713 info['filepath'] = filename
714 keep_video = None
715 for pp in self._pps:
716 try:
717 keep_video_wish, new_info = pp.run(info)
718 if keep_video_wish is not None:
719 if keep_video_wish:
720 keep_video = keep_video_wish
721 elif keep_video is None:
722 # No clear decision yet, let IE decide
723 keep_video = keep_video_wish
724 except PostProcessingError as e:
725 self.report_error(e.msg)
726 if keep_video is False and not self.params.get('keepvideo', False):
727 try:
728 self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
729 os.remove(encodeFilename(filename))
730 except (IOError, OSError):
731 self.report_warning(u'Unable to remove downloaded video file')
732
733 def in_download_archive(self, info_dict):
734 fn = self.params.get('download_archive')
735 if fn is None:
736 return False
737 vid_id = info_dict['extractor'] + u' ' + info_dict['id']
738 try:
739 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
740 for line in archive_file:
741 if line.strip() == vid_id:
742 return True
743 except IOError as ioe:
744 if ioe.errno != errno.ENOENT:
745 raise
746 return False
747
748 def record_download_archive(self, info_dict):
749 fn = self.params.get('download_archive')
750 if fn is None:
751 return
752 vid_id = info_dict['extractor'] + u' ' + info_dict['id']
753 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
754 archive_file.write(vid_id + u'\n')
755
756 @staticmethod
757 def format_resolution(format):
758 if format.get('height') is not None:
759 if format.get('width') is not None:
760 res = u'%sx%s' % (format['width'], format['height'])
761 else:
762 res = u'%sp' % format['height']
763 else:
764 res = '???'
765 return res
766
767 def list_formats(self, info_dict):
768 formats_s = []
769 for format in info_dict.get('formats', [info_dict]):
770 formats_s.append(u'%-15s: %-5s %-15s[%s]' % (
771 format['format_id'],
772 format['ext'],
773 format.get('format_note') or '-',
774 self.format_resolution(format),
775 )
776 )
777 if len(formats_s) != 1:
778 formats_s[0] += ' (worst)'
779 formats_s[-1] += ' (best)'
780 formats_s = "\n".join(formats_s)
781 self.to_screen(u'[info] Available formats for %s:\n'
782 u'format code extension note resolution\n%s' % (
783 info_dict['id'], formats_s))