]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/FileDownloader.py
debian/watch: Download version specific for distributors.
[youtubedl] / youtube_dl / FileDownloader.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import absolute_import
5
6 import math
7 import io
8 import os
9 import re
10 import shutil
11 import socket
12 import subprocess
13 import sys
14 import time
15 import traceback
16
17 if os.name == 'nt':
18 import ctypes
19
20 from .utils import *
21 from .InfoExtractors import get_info_extractor
22
23
24 class FileDownloader(object):
25 """File Downloader class.
26
27 File downloader objects are the ones responsible of downloading the
28 actual video file and writing it to disk if the user has requested
29 it, among some other tasks. In most cases there should be one per
30 program. As, given a video URL, the downloader doesn't know how to
31 extract all the needed information, task that InfoExtractors do, it
32 has to pass the URL to one of them.
33
34 For this, file downloader objects have a method that allows
35 InfoExtractors to be registered in a given order. When it is passed
36 a URL, the file downloader handles it to the first InfoExtractor it
37 finds that reports being able to handle it. The InfoExtractor extracts
38 all the information about the video or videos the URL refers to, and
39 asks the FileDownloader to process the video information, possibly
40 downloading the video.
41
42 File downloaders accept a lot of parameters. In order not to saturate
43 the object constructor with arguments, it receives a dictionary of
44 options instead. These options are available through the params
45 attribute for the InfoExtractors to use. The FileDownloader also
46 registers itself as the downloader in charge for the InfoExtractors
47 that are added to it, so this is a "mutual registration".
48
49 Available options:
50
51 username: Username for authentication purposes.
52 password: Password for authentication purposes.
53 usenetrc: Use netrc for authentication instead.
54 quiet: Do not print messages to stdout.
55 forceurl: Force printing final URL.
56 forcetitle: Force printing title.
57 forceid: Force printing ID.
58 forcethumbnail: Force printing thumbnail URL.
59 forcedescription: Force printing description.
60 forcefilename: Force printing final filename.
61 simulate: Do not download the video files.
62 format: Video format code.
63 format_limit: Highest quality format to try.
64 outtmpl: Template for output names.
65 restrictfilenames: Do not allow "&" and spaces in file names
66 ignoreerrors: Do not stop on download errors.
67 ratelimit: Download speed limit, in bytes/sec.
68 nooverwrites: Prevent overwriting files.
69 retries: Number of times to retry for HTTP error 5xx
70 buffersize: Size of download buffer in bytes.
71 noresizebuffer: Do not automatically resize the download buffer.
72 continuedl: Try to continue downloads if possible.
73 noprogress: Do not print the progress bar.
74 playliststart: Playlist item to start at.
75 playlistend: Playlist item to end at.
76 matchtitle: Download only matching titles.
77 rejecttitle: Reject downloads for matching titles.
78 logtostderr: Log messages to stderr instead of stdout.
79 consoletitle: Display progress in console window's titlebar.
80 nopart: Do not use temporary .part files.
81 updatetime: Use the Last-modified header to set output file timestamps.
82 writedescription: Write the video description to a .description file
83 writeinfojson: Write the video description to a .info.json file
84 writethumbnail: Write the thumbnail image to a file
85 writesubtitles: Write the video subtitles to a file
86 allsubtitles: Downloads all the subtitles of the video
87 listsubtitles: Lists all available subtitles for the video
88 subtitlesformat: Subtitle format [sbv/srt] (default=srt)
89 subtitleslang: Language of the subtitles to download
90 test: Download only first bytes to test the downloader.
91 keepvideo: Keep the video file after post-processing
92 min_filesize: Skip files smaller than this size
93 max_filesize: Skip files larger than this size
94 daterange: A DateRange object, download only if the upload_date is in the range.
95 skip_download: Skip the actual download of the video file
96 """
97
98 params = None
99 _ies = []
100 _pps = []
101 _download_retcode = None
102 _num_downloads = None
103 _screen_file = None
104
105 def __init__(self, params):
106 """Create a FileDownloader object with the given options."""
107 self._ies = []
108 self._pps = []
109 self._progress_hooks = []
110 self._download_retcode = 0
111 self._num_downloads = 0
112 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
113 self.params = params
114
115 if '%(stitle)s' in self.params['outtmpl']:
116 self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
117
118 @staticmethod
119 def format_bytes(bytes):
120 if bytes is None:
121 return 'N/A'
122 if type(bytes) is str:
123 bytes = float(bytes)
124 if bytes == 0.0:
125 exponent = 0
126 else:
127 exponent = int(math.log(bytes, 1024.0))
128 suffix = ['B','KiB','MiB','GiB','TiB','PiB','EiB','ZiB','YiB'][exponent]
129 converted = float(bytes) / float(1024 ** exponent)
130 return '%.2f%s' % (converted, suffix)
131
132 @staticmethod
133 def calc_percent(byte_counter, data_len):
134 if data_len is None:
135 return '---.-%'
136 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
137
138 @staticmethod
139 def calc_eta(start, now, total, current):
140 if total is None:
141 return '--:--'
142 dif = now - start
143 if current == 0 or dif < 0.001: # One millisecond
144 return '--:--'
145 rate = float(current) / dif
146 eta = int((float(total) - float(current)) / rate)
147 (eta_mins, eta_secs) = divmod(eta, 60)
148 if eta_mins > 99:
149 return '--:--'
150 return '%02d:%02d' % (eta_mins, eta_secs)
151
152 @staticmethod
153 def calc_speed(start, now, bytes):
154 dif = now - start
155 if bytes == 0 or dif < 0.001: # One millisecond
156 return '%10s' % '---b/s'
157 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
158
159 @staticmethod
160 def best_block_size(elapsed_time, bytes):
161 new_min = max(bytes / 2.0, 1.0)
162 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
163 if elapsed_time < 0.001:
164 return int(new_max)
165 rate = bytes / elapsed_time
166 if rate > new_max:
167 return int(new_max)
168 if rate < new_min:
169 return int(new_min)
170 return int(rate)
171
172 @staticmethod
173 def parse_bytes(bytestr):
174 """Parse a string indicating a byte quantity into an integer."""
175 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
176 if matchobj is None:
177 return None
178 number = float(matchobj.group(1))
179 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
180 return int(round(number * multiplier))
181
182 def add_info_extractor(self, ie):
183 """Add an InfoExtractor object to the end of the list."""
184 self._ies.append(ie)
185 ie.set_downloader(self)
186
187 def add_post_processor(self, pp):
188 """Add a PostProcessor object to the end of the chain."""
189 self._pps.append(pp)
190 pp.set_downloader(self)
191
192 def to_screen(self, message, skip_eol=False):
193 """Print message to stdout if not in quiet mode."""
194 assert type(message) == type(u'')
195 if not self.params.get('quiet', False):
196 terminator = [u'\n', u''][skip_eol]
197 output = message + terminator
198 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
199 output = output.encode(preferredencoding(), 'ignore')
200 self._screen_file.write(output)
201 self._screen_file.flush()
202
203 def to_stderr(self, message):
204 """Print message to stderr."""
205 assert type(message) == type(u'')
206 output = message + u'\n'
207 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
208 output = output.encode(preferredencoding())
209 sys.stderr.write(output)
210
211 def to_cons_title(self, message):
212 """Set console/terminal window title to message."""
213 if not self.params.get('consoletitle', False):
214 return
215 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
216 # c_wchar_p() might not be necessary if `message` is
217 # already of type unicode()
218 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
219 elif 'TERM' in os.environ:
220 self.to_screen('\033]0;%s\007' % message, skip_eol=True)
221
222 def fixed_template(self):
223 """Checks if the output template is fixed."""
224 return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
225
226 def trouble(self, message=None, tb=None):
227 """Determine action to take when a download problem appears.
228
229 Depending on if the downloader has been configured to ignore
230 download errors or not, this method may throw an exception or
231 not when errors are found, after printing the message.
232
233 tb, if given, is additional traceback information.
234 """
235 if message is not None:
236 self.to_stderr(message)
237 if self.params.get('verbose'):
238 if tb is None:
239 if sys.exc_info()[0]: # if .trouble has been called from an except block
240 tb = u''
241 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
242 tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
243 tb += compat_str(traceback.format_exc())
244 else:
245 tb_data = traceback.format_list(traceback.extract_stack())
246 tb = u''.join(tb_data)
247 self.to_stderr(tb)
248 if not self.params.get('ignoreerrors', False):
249 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
250 exc_info = sys.exc_info()[1].exc_info
251 else:
252 exc_info = sys.exc_info()
253 raise DownloadError(message, exc_info)
254 self._download_retcode = 1
255
256 def report_warning(self, message):
257 '''
258 Print the message to stderr, it will be prefixed with 'WARNING:'
259 If stderr is a tty file the 'WARNING:' will be colored
260 '''
261 if sys.stderr.isatty() and os.name != 'nt':
262 _msg_header=u'\033[0;33mWARNING:\033[0m'
263 else:
264 _msg_header=u'WARNING:'
265 warning_message=u'%s %s' % (_msg_header,message)
266 self.to_stderr(warning_message)
267
268 def report_error(self, message, tb=None):
269 '''
270 Do the same as trouble, but prefixes the message with 'ERROR:', colored
271 in red if stderr is a tty file.
272 '''
273 if sys.stderr.isatty() and os.name != 'nt':
274 _msg_header = u'\033[0;31mERROR:\033[0m'
275 else:
276 _msg_header = u'ERROR:'
277 error_message = u'%s %s' % (_msg_header, message)
278 self.trouble(error_message, tb)
279
280 def slow_down(self, start_time, byte_counter):
281 """Sleep if the download speed is over the rate limit."""
282 rate_limit = self.params.get('ratelimit', None)
283 if rate_limit is None or byte_counter == 0:
284 return
285 now = time.time()
286 elapsed = now - start_time
287 if elapsed <= 0.0:
288 return
289 speed = float(byte_counter) / elapsed
290 if speed > rate_limit:
291 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
292
293 def temp_name(self, filename):
294 """Returns a temporary filename for the given filename."""
295 if self.params.get('nopart', False) or filename == u'-' or \
296 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
297 return filename
298 return filename + u'.part'
299
300 def undo_temp_name(self, filename):
301 if filename.endswith(u'.part'):
302 return filename[:-len(u'.part')]
303 return filename
304
305 def try_rename(self, old_filename, new_filename):
306 try:
307 if old_filename == new_filename:
308 return
309 os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
310 except (IOError, OSError) as err:
311 self.report_error(u'unable to rename file')
312
313 def try_utime(self, filename, last_modified_hdr):
314 """Try to set the last-modified time of the given file."""
315 if last_modified_hdr is None:
316 return
317 if not os.path.isfile(encodeFilename(filename)):
318 return
319 timestr = last_modified_hdr
320 if timestr is None:
321 return
322 filetime = timeconvert(timestr)
323 if filetime is None:
324 return filetime
325 try:
326 os.utime(filename, (time.time(), filetime))
327 except:
328 pass
329 return filetime
330
331 def report_writedescription(self, descfn):
332 """ Report that the description file is being written """
333 self.to_screen(u'[info] Writing video description to: ' + descfn)
334
335 def report_writesubtitles(self, sub_filename):
336 """ Report that the subtitles file is being written """
337 self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
338
339 def report_writeinfojson(self, infofn):
340 """ Report that the metadata file has been written """
341 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
342
343 def report_destination(self, filename):
344 """Report destination filename."""
345 self.to_screen(u'[download] Destination: ' + filename)
346
347 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
348 """Report download progress."""
349 if self.params.get('noprogress', False):
350 return
351 clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
352 if self.params.get('progress_with_newline', False):
353 self.to_screen(u'[download] %s of %s at %s ETA %s' %
354 (percent_str, data_len_str, speed_str, eta_str))
355 else:
356 self.to_screen(u'\r%s[download] %s of %s at %s ETA %s' %
357 (clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
358 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
359 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
360
361 def report_resuming_byte(self, resume_len):
362 """Report attempt to resume at given byte."""
363 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
364
365 def report_retry(self, count, retries):
366 """Report retry in case of HTTP error 5xx"""
367 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
368
369 def report_file_already_downloaded(self, file_name):
370 """Report file has already been fully downloaded."""
371 try:
372 self.to_screen(u'[download] %s has already been downloaded' % file_name)
373 except (UnicodeEncodeError) as err:
374 self.to_screen(u'[download] The file has already been downloaded')
375
376 def report_unable_to_resume(self):
377 """Report it was impossible to resume download."""
378 self.to_screen(u'[download] Unable to resume')
379
380 def report_finish(self):
381 """Report download finished."""
382 if self.params.get('noprogress', False):
383 self.to_screen(u'[download] Download completed')
384 else:
385 self.to_screen(u'')
386
387 def increment_downloads(self):
388 """Increment the ordinal that assigns a number to each file."""
389 self._num_downloads += 1
390
391 def prepare_filename(self, info_dict):
392 """Generate the output filename."""
393 try:
394 template_dict = dict(info_dict)
395
396 template_dict['epoch'] = int(time.time())
397 autonumber_size = self.params.get('autonumber_size')
398 if autonumber_size is None:
399 autonumber_size = 5
400 autonumber_templ = u'%0' + str(autonumber_size) + u'd'
401 template_dict['autonumber'] = autonumber_templ % self._num_downloads
402 if template_dict['playlist_index'] is not None:
403 template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
404
405 sanitize = lambda k,v: sanitize_filename(
406 u'NA' if v is None else compat_str(v),
407 restricted=self.params.get('restrictfilenames'),
408 is_id=(k==u'id'))
409 template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
410
411 filename = self.params['outtmpl'] % template_dict
412 return filename
413 except KeyError as err:
414 self.report_error(u'Erroneous output template')
415 return None
416 except ValueError as err:
417 self.report_error(u'Insufficient system charset ' + repr(preferredencoding()))
418 return None
419
420 def _match_entry(self, info_dict):
421 """ Returns None iff the file should be downloaded """
422
423 title = info_dict['title']
424 matchtitle = self.params.get('matchtitle', False)
425 if matchtitle:
426 if not re.search(matchtitle, title, re.IGNORECASE):
427 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
428 rejecttitle = self.params.get('rejecttitle', False)
429 if rejecttitle:
430 if re.search(rejecttitle, title, re.IGNORECASE):
431 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
432 date = info_dict.get('upload_date', None)
433 if date is not None:
434 dateRange = self.params.get('daterange', DateRange())
435 if date not in dateRange:
436 return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
437 return None
438
439 def extract_info(self, url, download=True, ie_key=None, extra_info={}):
440 '''
441 Returns a list with a dictionary for each video we find.
442 If 'download', also downloads the videos.
443 extra_info is a dict containing the extra values to add to each result
444 '''
445
446 if ie_key:
447 ie = get_info_extractor(ie_key)()
448 ie.set_downloader(self)
449 ies = [ie]
450 else:
451 ies = self._ies
452
453 for ie in ies:
454 if not ie.suitable(url):
455 continue
456
457 if not ie.working():
458 self.report_warning(u'The program functionality for this site has been marked as broken, '
459 u'and will probably not work.')
460
461 try:
462 ie_result = ie.extract(url)
463 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
464 break
465 if isinstance(ie_result, list):
466 # Backwards compatibility: old IE result format
467 for result in ie_result:
468 result.update(extra_info)
469 ie_result = {
470 '_type': 'compat_list',
471 'entries': ie_result,
472 }
473 else:
474 ie_result.update(extra_info)
475 if 'extractor' not in ie_result:
476 ie_result['extractor'] = ie.IE_NAME
477 return self.process_ie_result(ie_result, download=download)
478 except ExtractorError as de: # An error we somewhat expected
479 self.report_error(compat_str(de), de.format_traceback())
480 break
481 except Exception as e:
482 if self.params.get('ignoreerrors', False):
483 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
484 break
485 else:
486 raise
487 else:
488 self.report_error(u'no suitable InfoExtractor: %s' % url)
489
490 def process_ie_result(self, ie_result, download=True, extra_info={}):
491 """
492 Take the result of the ie(may be modified) and resolve all unresolved
493 references (URLs, playlist items).
494
495 It will also download the videos if 'download'.
496 Returns the resolved ie_result.
497 """
498
499 result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
500 if result_type == 'video':
501 if 'playlist' not in ie_result:
502 # It isn't part of a playlist
503 ie_result['playlist'] = None
504 ie_result['playlist_index'] = None
505 if download:
506 self.process_info(ie_result)
507 return ie_result
508 elif result_type == 'url':
509 # We have to add extra_info to the results because it may be
510 # contained in a playlist
511 return self.extract_info(ie_result['url'],
512 download,
513 ie_key=ie_result.get('ie_key'),
514 extra_info=extra_info)
515 elif result_type == 'playlist':
516 # We process each entry in the playlist
517 playlist = ie_result.get('title', None) or ie_result.get('id', None)
518 self.to_screen(u'[download] Downloading playlist: %s' % playlist)
519
520 playlist_results = []
521
522 n_all_entries = len(ie_result['entries'])
523 playliststart = self.params.get('playliststart', 1) - 1
524 playlistend = self.params.get('playlistend', -1)
525
526 if playlistend == -1:
527 entries = ie_result['entries'][playliststart:]
528 else:
529 entries = ie_result['entries'][playliststart:playlistend]
530
531 n_entries = len(entries)
532
533 self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
534 (ie_result['extractor'], playlist, n_all_entries, n_entries))
535
536 for i,entry in enumerate(entries,1):
537 self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
538 extra = {
539 'playlist': playlist,
540 'playlist_index': i + playliststart,
541 }
542 entry_result = self.process_ie_result(entry,
543 download=download,
544 extra_info=extra)
545 playlist_results.append(entry_result)
546 ie_result['entries'] = playlist_results
547 return ie_result
548 elif result_type == 'compat_list':
549 def _fixup(r):
550 r.setdefault('extractor', ie_result['extractor'])
551 return r
552 ie_result['entries'] = [
553 self.process_ie_result(_fixup(r), download=download)
554 for r in ie_result['entries']
555 ]
556 return ie_result
557 else:
558 raise Exception('Invalid result type: %s' % result_type)
559
560 def process_info(self, info_dict):
561 """Process a single resolved IE result."""
562
563 assert info_dict.get('_type', 'video') == 'video'
564 #We increment the download the download count here to match the previous behaviour.
565 self.increment_downloads()
566
567 info_dict['fulltitle'] = info_dict['title']
568 if len(info_dict['title']) > 200:
569 info_dict['title'] = info_dict['title'][:197] + u'...'
570
571 # Keep for backwards compatibility
572 info_dict['stitle'] = info_dict['title']
573
574 if not 'format' in info_dict:
575 info_dict['format'] = info_dict['ext']
576
577 reason = self._match_entry(info_dict)
578 if reason is not None:
579 self.to_screen(u'[download] ' + reason)
580 return
581
582 max_downloads = self.params.get('max_downloads')
583 if max_downloads is not None:
584 if self._num_downloads > int(max_downloads):
585 raise MaxDownloadsReached()
586
587 filename = self.prepare_filename(info_dict)
588
589 # Forced printings
590 if self.params.get('forcetitle', False):
591 compat_print(info_dict['title'])
592 if self.params.get('forceid', False):
593 compat_print(info_dict['id'])
594 if self.params.get('forceurl', False):
595 compat_print(info_dict['url'])
596 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
597 compat_print(info_dict['thumbnail'])
598 if self.params.get('forcedescription', False) and 'description' in info_dict:
599 compat_print(info_dict['description'])
600 if self.params.get('forcefilename', False) and filename is not None:
601 compat_print(filename)
602 if self.params.get('forceformat', False):
603 compat_print(info_dict['format'])
604
605 # Do nothing else if in simulate mode
606 if self.params.get('simulate', False):
607 return
608
609 if filename is None:
610 return
611
612 try:
613 dn = os.path.dirname(encodeFilename(filename))
614 if dn != '' and not os.path.exists(dn):
615 os.makedirs(dn)
616 except (OSError, IOError) as err:
617 self.report_error(u'unable to create directory ' + compat_str(err))
618 return
619
620 if self.params.get('writedescription', False):
621 try:
622 descfn = filename + u'.description'
623 self.report_writedescription(descfn)
624 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
625 descfile.write(info_dict['description'])
626 except (OSError, IOError):
627 self.report_error(u'Cannot write description file ' + descfn)
628 return
629
630 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
631 # subtitles download errors are already managed as troubles in relevant IE
632 # that way it will silently go on when used with unsupporting IE
633 subtitle = info_dict['subtitles'][0]
634 (sub_error, sub_lang, sub) = subtitle
635 sub_format = self.params.get('subtitlesformat')
636 if sub_error:
637 self.report_warning("Some error while getting the subtitles")
638 else:
639 try:
640 sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
641 self.report_writesubtitles(sub_filename)
642 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
643 subfile.write(sub)
644 except (OSError, IOError):
645 self.report_error(u'Cannot write subtitles file ' + descfn)
646 return
647
648 if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
649 subtitles = info_dict['subtitles']
650 sub_format = self.params.get('subtitlesformat')
651 for subtitle in subtitles:
652 (sub_error, sub_lang, sub) = subtitle
653 if sub_error:
654 self.report_warning("Some error while getting the subtitles")
655 else:
656 try:
657 sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
658 self.report_writesubtitles(sub_filename)
659 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
660 subfile.write(sub)
661 except (OSError, IOError):
662 self.report_error(u'Cannot write subtitles file ' + descfn)
663 return
664
665 if self.params.get('writeinfojson', False):
666 infofn = filename + u'.info.json'
667 self.report_writeinfojson(infofn)
668 try:
669 json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
670 write_json_file(json_info_dict, encodeFilename(infofn))
671 except (OSError, IOError):
672 self.report_error(u'Cannot write metadata to JSON file ' + infofn)
673 return
674
675 if self.params.get('writethumbnail', False):
676 if 'thumbnail' in info_dict:
677 thumb_format = info_dict['thumbnail'].rpartition(u'/')[2].rpartition(u'.')[2]
678 if not thumb_format:
679 thumb_format = 'jpg'
680 thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
681 self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
682 (info_dict['extractor'], info_dict['id']))
683 uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
684 with open(thumb_filename, 'wb') as thumbf:
685 shutil.copyfileobj(uf, thumbf)
686 self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
687 (info_dict['extractor'], info_dict['id'], thumb_filename))
688
689 if not self.params.get('skip_download', False):
690 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
691 success = True
692 else:
693 try:
694 success = self._do_download(filename, info_dict)
695 except (OSError, IOError) as err:
696 raise UnavailableVideoError()
697 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
698 self.report_error(u'unable to download video data: %s' % str(err))
699 return
700 except (ContentTooShortError, ) as err:
701 self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
702 return
703
704 if success:
705 try:
706 self.post_process(filename, info_dict)
707 except (PostProcessingError) as err:
708 self.report_error(u'postprocessing: %s' % str(err))
709 return
710
711 def download(self, url_list):
712 """Download a given list of URLs."""
713 if len(url_list) > 1 and self.fixed_template():
714 raise SameFileError(self.params['outtmpl'])
715
716 for url in url_list:
717 try:
718 #It also downloads the videos
719 videos = self.extract_info(url)
720 except UnavailableVideoError:
721 self.report_error(u'unable to download video')
722 except MaxDownloadsReached:
723 self.to_screen(u'[info] Maximum number of downloaded files reached.')
724 raise
725
726 return self._download_retcode
727
728 def post_process(self, filename, ie_info):
729 """Run all the postprocessors on the given file."""
730 info = dict(ie_info)
731 info['filepath'] = filename
732 keep_video = None
733 for pp in self._pps:
734 try:
735 keep_video_wish,new_info = pp.run(info)
736 if keep_video_wish is not None:
737 if keep_video_wish:
738 keep_video = keep_video_wish
739 elif keep_video is None:
740 # No clear decision yet, let IE decide
741 keep_video = keep_video_wish
742 except PostProcessingError as e:
743 self.to_stderr(u'ERROR: ' + e.msg)
744 if keep_video is False and not self.params.get('keepvideo', False):
745 try:
746 self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
747 os.remove(encodeFilename(filename))
748 except (IOError, OSError):
749 self.report_warning(u'Unable to remove downloaded video file')
750
751 def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):
752 self.report_destination(filename)
753 tmpfilename = self.temp_name(filename)
754
755 # Check for rtmpdump first
756 try:
757 subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
758 except (OSError, IOError):
759 self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
760 return False
761
762 # Download using rtmpdump. rtmpdump returns exit code 2 when
763 # the connection was interrumpted and resuming appears to be
764 # possible. This is part of rtmpdump's normal usage, AFAIK.
765 basic_args = ['rtmpdump', '-q', '-r', url, '-o', tmpfilename]
766 if self.params.get('verbose', False): basic_args[1] = '-v'
767 if player_url is not None:
768 basic_args += ['-W', player_url]
769 if page_url is not None:
770 basic_args += ['--pageUrl', page_url]
771 if play_path is not None:
772 basic_args += ['-y', play_path]
773 if tc_url is not None:
774 basic_args += ['--tcUrl', url]
775 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
776 if self.params.get('verbose', False):
777 try:
778 import pipes
779 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
780 except ImportError:
781 shell_quote = repr
782 self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
783 retval = subprocess.call(args)
784 while retval == 2 or retval == 1:
785 prevsize = os.path.getsize(encodeFilename(tmpfilename))
786 self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
787 time.sleep(5.0) # This seems to be needed
788 retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
789 cursize = os.path.getsize(encodeFilename(tmpfilename))
790 if prevsize == cursize and retval == 1:
791 break
792 # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
793 if prevsize == cursize and retval == 2 and cursize > 1024:
794 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
795 retval = 0
796 break
797 if retval == 0:
798 fsize = os.path.getsize(encodeFilename(tmpfilename))
799 self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
800 self.try_rename(tmpfilename, filename)
801 self._hook_progress({
802 'downloaded_bytes': fsize,
803 'total_bytes': fsize,
804 'filename': filename,
805 'status': 'finished',
806 })
807 return True
808 else:
809 self.to_stderr(u"\n")
810 self.report_error(u'rtmpdump exited with code %d' % retval)
811 return False
812
813 def _do_download(self, filename, info_dict):
814 url = info_dict['url']
815
816 # Check file already present
817 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
818 self.report_file_already_downloaded(filename)
819 self._hook_progress({
820 'filename': filename,
821 'status': 'finished',
822 })
823 return True
824
825 # Attempt to download using rtmpdump
826 if url.startswith('rtmp'):
827 return self._download_with_rtmpdump(filename, url,
828 info_dict.get('player_url', None),
829 info_dict.get('page_url', None),
830 info_dict.get('play_path', None),
831 info_dict.get('tc_url', None))
832
833 tmpfilename = self.temp_name(filename)
834 stream = None
835
836 # Do not include the Accept-Encoding header
837 headers = {'Youtubedl-no-compression': 'True'}
838 if 'user_agent' in info_dict:
839 headers['Youtubedl-user-agent'] = info_dict['user_agent']
840 basic_request = compat_urllib_request.Request(url, None, headers)
841 request = compat_urllib_request.Request(url, None, headers)
842
843 if self.params.get('test', False):
844 request.add_header('Range','bytes=0-10240')
845
846 # Establish possible resume length
847 if os.path.isfile(encodeFilename(tmpfilename)):
848 resume_len = os.path.getsize(encodeFilename(tmpfilename))
849 else:
850 resume_len = 0
851
852 open_mode = 'wb'
853 if resume_len != 0:
854 if self.params.get('continuedl', False):
855 self.report_resuming_byte(resume_len)
856 request.add_header('Range','bytes=%d-' % resume_len)
857 open_mode = 'ab'
858 else:
859 resume_len = 0
860
861 count = 0
862 retries = self.params.get('retries', 0)
863 while count <= retries:
864 # Establish connection
865 try:
866 if count == 0 and 'urlhandle' in info_dict:
867 data = info_dict['urlhandle']
868 data = compat_urllib_request.urlopen(request)
869 break
870 except (compat_urllib_error.HTTPError, ) as err:
871 if (err.code < 500 or err.code >= 600) and err.code != 416:
872 # Unexpected HTTP error
873 raise
874 elif err.code == 416:
875 # Unable to resume (requested range not satisfiable)
876 try:
877 # Open the connection again without the range header
878 data = compat_urllib_request.urlopen(basic_request)
879 content_length = data.info()['Content-Length']
880 except (compat_urllib_error.HTTPError, ) as err:
881 if err.code < 500 or err.code >= 600:
882 raise
883 else:
884 # Examine the reported length
885 if (content_length is not None and
886 (resume_len - 100 < int(content_length) < resume_len + 100)):
887 # The file had already been fully downloaded.
888 # Explanation to the above condition: in issue #175 it was revealed that
889 # YouTube sometimes adds or removes a few bytes from the end of the file,
890 # changing the file size slightly and causing problems for some users. So
891 # I decided to implement a suggested change and consider the file
892 # completely downloaded if the file size differs less than 100 bytes from
893 # the one in the hard drive.
894 self.report_file_already_downloaded(filename)
895 self.try_rename(tmpfilename, filename)
896 self._hook_progress({
897 'filename': filename,
898 'status': 'finished',
899 })
900 return True
901 else:
902 # The length does not match, we start the download over
903 self.report_unable_to_resume()
904 open_mode = 'wb'
905 break
906 # Retry
907 count += 1
908 if count <= retries:
909 self.report_retry(count, retries)
910
911 if count > retries:
912 self.report_error(u'giving up after %s retries' % retries)
913 return False
914
915 data_len = data.info().get('Content-length', None)
916 if data_len is not None:
917 data_len = int(data_len) + resume_len
918 min_data_len = self.params.get("min_filesize", None)
919 max_data_len = self.params.get("max_filesize", None)
920 if min_data_len is not None and data_len < min_data_len:
921 self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
922 return False
923 if max_data_len is not None and data_len > max_data_len:
924 self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
925 return False
926
927 data_len_str = self.format_bytes(data_len)
928 byte_counter = 0 + resume_len
929 block_size = self.params.get('buffersize', 1024)
930 start = time.time()
931 while True:
932 # Download and write
933 before = time.time()
934 data_block = data.read(block_size)
935 after = time.time()
936 if len(data_block) == 0:
937 break
938 byte_counter += len(data_block)
939
940 # Open file just in time
941 if stream is None:
942 try:
943 (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
944 assert stream is not None
945 filename = self.undo_temp_name(tmpfilename)
946 self.report_destination(filename)
947 except (OSError, IOError) as err:
948 self.report_error(u'unable to open for writing: %s' % str(err))
949 return False
950 try:
951 stream.write(data_block)
952 except (IOError, OSError) as err:
953 self.to_stderr(u"\n")
954 self.report_error(u'unable to write data: %s' % str(err))
955 return False
956 if not self.params.get('noresizebuffer', False):
957 block_size = self.best_block_size(after - before, len(data_block))
958
959 # Progress message
960 speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
961 if data_len is None:
962 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
963 else:
964 percent_str = self.calc_percent(byte_counter, data_len)
965 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
966 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
967
968 self._hook_progress({
969 'downloaded_bytes': byte_counter,
970 'total_bytes': data_len,
971 'tmpfilename': tmpfilename,
972 'filename': filename,
973 'status': 'downloading',
974 })
975
976 # Apply rate limit
977 self.slow_down(start, byte_counter - resume_len)
978
979 if stream is None:
980 self.to_stderr(u"\n")
981 self.report_error(u'Did not get any data blocks')
982 return False
983 stream.close()
984 self.report_finish()
985 if data_len is not None and byte_counter != data_len:
986 raise ContentTooShortError(byte_counter, int(data_len))
987 self.try_rename(tmpfilename, filename)
988
989 # Update file modification time
990 if self.params.get('updatetime', True):
991 info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
992
993 self._hook_progress({
994 'downloaded_bytes': byte_counter,
995 'total_bytes': byte_counter,
996 'filename': filename,
997 'status': 'finished',
998 })
999
1000 return True
1001
1002 def _hook_progress(self, status):
1003 for ph in self._progress_hooks:
1004 ph(status)
1005
1006 def add_progress_hook(self, ph):
1007 """ ph gets called on download progress, with a dictionary with the entries
1008 * filename: The final filename
1009 * status: One of "downloading" and "finished"
1010
1011 It can also have some of the following entries:
1012
1013 * downloaded_bytes: Bytes on disks
1014 * total_bytes: Total bytes, None if unknown
1015 * tmpfilename: The filename we're currently writing to
1016
1017 Hooks are guaranteed to be called at least once (with status "finished")
1018 if the download is successful.
1019 """
1020 self._progress_hooks.append(ph)