]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/downloader/common.py
5d66211476c521c80054558b5d1134ac4179a828
[youtubedl] / youtube_dl / downloader / common.py
1 from __future__ import division, unicode_literals
2
3 import os
4 import re
5 import sys
6 import time
7 import random
8
9 from ..compat import compat_os_name
10 from ..utils import (
11 encodeFilename,
12 error_to_compat_str,
13 decodeArgument,
14 format_bytes,
15 timeconvert,
16 )
17
18
19 class FileDownloader(object):
20 """File Downloader class.
21
22 File downloader objects are the ones responsible of downloading the
23 actual video file and writing it to disk.
24
25 File downloaders accept a lot of parameters. In order not to saturate
26 the object constructor with arguments, it receives a dictionary of
27 options instead.
28
29 Available options:
30
31 verbose: Print additional info to stdout.
32 quiet: Do not print messages to stdout.
33 ratelimit: Download speed limit, in bytes/sec.
34 retries: Number of times to retry for HTTP error 5xx
35 buffersize: Size of download buffer in bytes.
36 noresizebuffer: Do not automatically resize the download buffer.
37 continuedl: Try to continue downloads if possible.
38 noprogress: Do not print the progress bar.
39 logtostderr: Log messages to stderr instead of stdout.
40 consoletitle: Display progress in console window's titlebar.
41 nopart: Do not use temporary .part files.
42 updatetime: Use the Last-modified header to set output file timestamps.
43 test: Download only first bytes to test the downloader.
44 min_filesize: Skip files smaller than this size
45 max_filesize: Skip files larger than this size
46 xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
47 (experimental)
48 external_downloader_args: A list of additional command-line arguments for the
49 external downloader.
50 hls_use_mpegts: Use the mpegts container for HLS videos.
51
52 Subclasses of this one must re-define the real_download method.
53 """
54
55 _TEST_FILE_SIZE = 10241
56 params = None
57
58 def __init__(self, ydl, params):
59 """Create a FileDownloader object with the given options."""
60 self.ydl = ydl
61 self._progress_hooks = []
62 self.params = params
63 self.add_progress_hook(self.report_progress)
64
65 @staticmethod
66 def format_seconds(seconds):
67 (mins, secs) = divmod(seconds, 60)
68 (hours, mins) = divmod(mins, 60)
69 if hours > 99:
70 return '--:--:--'
71 if hours == 0:
72 return '%02d:%02d' % (mins, secs)
73 else:
74 return '%02d:%02d:%02d' % (hours, mins, secs)
75
76 @staticmethod
77 def calc_percent(byte_counter, data_len):
78 if data_len is None:
79 return None
80 return float(byte_counter) / float(data_len) * 100.0
81
82 @staticmethod
83 def format_percent(percent):
84 if percent is None:
85 return '---.-%'
86 return '%6s' % ('%3.1f%%' % percent)
87
88 @staticmethod
89 def calc_eta(start, now, total, current):
90 if total is None:
91 return None
92 if now is None:
93 now = time.time()
94 dif = now - start
95 if current == 0 or dif < 0.001: # One millisecond
96 return None
97 rate = float(current) / dif
98 return int((float(total) - float(current)) / rate)
99
100 @staticmethod
101 def format_eta(eta):
102 if eta is None:
103 return '--:--'
104 return FileDownloader.format_seconds(eta)
105
106 @staticmethod
107 def calc_speed(start, now, bytes):
108 dif = now - start
109 if bytes == 0 or dif < 0.001: # One millisecond
110 return None
111 return float(bytes) / dif
112
113 @staticmethod
114 def format_speed(speed):
115 if speed is None:
116 return '%10s' % '---b/s'
117 return '%10s' % ('%s/s' % format_bytes(speed))
118
119 @staticmethod
120 def format_retries(retries):
121 return 'inf' if retries == float('inf') else '%.0f' % retries
122
123 @staticmethod
124 def best_block_size(elapsed_time, bytes):
125 new_min = max(bytes / 2.0, 1.0)
126 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
127 if elapsed_time < 0.001:
128 return int(new_max)
129 rate = bytes / elapsed_time
130 if rate > new_max:
131 return int(new_max)
132 if rate < new_min:
133 return int(new_min)
134 return int(rate)
135
136 @staticmethod
137 def parse_bytes(bytestr):
138 """Parse a string indicating a byte quantity into an integer."""
139 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
140 if matchobj is None:
141 return None
142 number = float(matchobj.group(1))
143 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
144 return int(round(number * multiplier))
145
146 def to_screen(self, *args, **kargs):
147 self.ydl.to_screen(*args, **kargs)
148
149 def to_stderr(self, message):
150 self.ydl.to_screen(message)
151
152 def to_console_title(self, message):
153 self.ydl.to_console_title(message)
154
155 def trouble(self, *args, **kargs):
156 self.ydl.trouble(*args, **kargs)
157
158 def report_warning(self, *args, **kargs):
159 self.ydl.report_warning(*args, **kargs)
160
161 def report_error(self, *args, **kargs):
162 self.ydl.report_error(*args, **kargs)
163
164 def slow_down(self, start_time, now, byte_counter):
165 """Sleep if the download speed is over the rate limit."""
166 rate_limit = self.params.get('ratelimit')
167 if rate_limit is None or byte_counter == 0:
168 return
169 if now is None:
170 now = time.time()
171 elapsed = now - start_time
172 if elapsed <= 0.0:
173 return
174 speed = float(byte_counter) / elapsed
175 if speed > rate_limit:
176 time.sleep(max((byte_counter // rate_limit) - elapsed, 0))
177
178 def temp_name(self, filename):
179 """Returns a temporary filename for the given filename."""
180 if self.params.get('nopart', False) or filename == '-' or \
181 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
182 return filename
183 return filename + '.part'
184
185 def undo_temp_name(self, filename):
186 if filename.endswith('.part'):
187 return filename[:-len('.part')]
188 return filename
189
190 def ytdl_filename(self, filename):
191 return filename + '.ytdl'
192
193 def try_rename(self, old_filename, new_filename):
194 try:
195 if old_filename == new_filename:
196 return
197 os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
198 except (IOError, OSError) as err:
199 self.report_error('unable to rename file: %s' % error_to_compat_str(err))
200
201 def try_utime(self, filename, last_modified_hdr):
202 """Try to set the last-modified time of the given file."""
203 if last_modified_hdr is None:
204 return
205 if not os.path.isfile(encodeFilename(filename)):
206 return
207 timestr = last_modified_hdr
208 if timestr is None:
209 return
210 filetime = timeconvert(timestr)
211 if filetime is None:
212 return filetime
213 # Ignore obviously invalid dates
214 if filetime == 0:
215 return
216 try:
217 os.utime(filename, (time.time(), filetime))
218 except Exception:
219 pass
220 return filetime
221
222 def report_destination(self, filename):
223 """Report destination filename."""
224 self.to_screen('[download] Destination: ' + filename)
225
226 def _report_progress_status(self, msg, is_last_line=False):
227 fullmsg = '[download] ' + msg
228 if self.params.get('progress_with_newline', False):
229 self.to_screen(fullmsg)
230 else:
231 if compat_os_name == 'nt':
232 prev_len = getattr(self, '_report_progress_prev_line_length',
233 0)
234 if prev_len > len(fullmsg):
235 fullmsg += ' ' * (prev_len - len(fullmsg))
236 self._report_progress_prev_line_length = len(fullmsg)
237 clear_line = '\r'
238 else:
239 clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
240 self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
241 self.to_console_title('youtube-dl ' + msg)
242
243 def report_progress(self, s):
244 if s['status'] == 'finished':
245 if self.params.get('noprogress', False):
246 self.to_screen('[download] Download completed')
247 else:
248 s['_total_bytes_str'] = format_bytes(s['total_bytes'])
249 if s.get('elapsed') is not None:
250 s['_elapsed_str'] = self.format_seconds(s['elapsed'])
251 msg_template = '100%% of %(_total_bytes_str)s in %(_elapsed_str)s'
252 else:
253 msg_template = '100%% of %(_total_bytes_str)s'
254 self._report_progress_status(
255 msg_template % s, is_last_line=True)
256
257 if self.params.get('noprogress'):
258 return
259
260 if s['status'] != 'downloading':
261 return
262
263 if s.get('eta') is not None:
264 s['_eta_str'] = self.format_eta(s['eta'])
265 else:
266 s['_eta_str'] = 'Unknown ETA'
267
268 if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
269 s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
270 elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
271 s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
272 else:
273 if s.get('downloaded_bytes') == 0:
274 s['_percent_str'] = self.format_percent(0)
275 else:
276 s['_percent_str'] = 'Unknown %'
277
278 if s.get('speed') is not None:
279 s['_speed_str'] = self.format_speed(s['speed'])
280 else:
281 s['_speed_str'] = 'Unknown speed'
282
283 if s.get('total_bytes') is not None:
284 s['_total_bytes_str'] = format_bytes(s['total_bytes'])
285 msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
286 elif s.get('total_bytes_estimate') is not None:
287 s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
288 msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
289 else:
290 if s.get('downloaded_bytes') is not None:
291 s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
292 if s.get('elapsed'):
293 s['_elapsed_str'] = self.format_seconds(s['elapsed'])
294 msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
295 else:
296 msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
297 else:
298 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
299
300 self._report_progress_status(msg_template % s)
301
302 def report_resuming_byte(self, resume_len):
303 """Report attempt to resume at given byte."""
304 self.to_screen('[download] Resuming download at byte %s' % resume_len)
305
306 def report_retry(self, count, retries):
307 """Report retry in case of HTTP error 5xx"""
308 self.to_screen(
309 '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
310 % (count, self.format_retries(retries)))
311
312 def report_file_already_downloaded(self, file_name):
313 """Report file has already been fully downloaded."""
314 try:
315 self.to_screen('[download] %s has already been downloaded' % file_name)
316 except UnicodeEncodeError:
317 self.to_screen('[download] The file has already been downloaded')
318
319 def report_unable_to_resume(self):
320 """Report it was impossible to resume download."""
321 self.to_screen('[download] Unable to resume')
322
323 def download(self, filename, info_dict):
324 """Download to a filename using the info from info_dict
325 Return True on success and False otherwise
326 """
327
328 nooverwrites_and_exists = (
329 self.params.get('nooverwrites', False) and
330 os.path.exists(encodeFilename(filename))
331 )
332
333 if not hasattr(filename, 'write'):
334 continuedl_and_exists = (
335 self.params.get('continuedl', True) and
336 os.path.isfile(encodeFilename(filename)) and
337 not self.params.get('nopart', False)
338 )
339
340 # Check file already present
341 if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
342 self.report_file_already_downloaded(filename)
343 self._hook_progress({
344 'filename': filename,
345 'status': 'finished',
346 'total_bytes': os.path.getsize(encodeFilename(filename)),
347 })
348 return True
349
350 min_sleep_interval = self.params.get('sleep_interval')
351 if min_sleep_interval:
352 max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
353 sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
354 self.to_screen(
355 '[download] Sleeping %s seconds...' % (
356 int(sleep_interval) if sleep_interval.is_integer()
357 else '%.2f' % sleep_interval))
358 time.sleep(sleep_interval)
359
360 return self.real_download(filename, info_dict)
361
362 def real_download(self, filename, info_dict):
363 """Real download process. Redefine in subclasses."""
364 raise NotImplementedError('This method must be implemented by subclasses')
365
366 def _hook_progress(self, status):
367 for ph in self._progress_hooks:
368 ph(status)
369
370 def add_progress_hook(self, ph):
371 # See YoutubeDl.py (search for progress_hooks) for a description of
372 # this interface
373 self._progress_hooks.append(ph)
374
375 def _debug_cmd(self, args, exe=None):
376 if not self.params.get('verbose', False):
377 return
378
379 str_args = [decodeArgument(a) for a in args]
380
381 if exe is None:
382 exe = os.path.basename(str_args[0])
383
384 try:
385 import pipes
386 shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
387 except ImportError:
388 shell_quote = repr
389 self.to_screen('[debug] %s command line: %s' % (
390 exe, shell_quote(str_args)))