]>
Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/downloader/common.py
14 class FileDownloader(object):
15 """File Downloader class.
17 File downloader objects are the ones responsible of downloading the
18 actual video file and writing it to disk.
20 File downloaders accept a lot of parameters. In order not to saturate
21 the object constructor with arguments, it receives a dictionary of
26 verbose: Print additional info to stdout.
27 quiet: Do not print messages to stdout.
28 ratelimit: Download speed limit, in bytes/sec.
29 retries: Number of times to retry for HTTP error 5xx
30 buffersize: Size of download buffer in bytes.
31 noresizebuffer: Do not automatically resize the download buffer.
32 continuedl: Try to continue downloads if possible.
33 noprogress: Do not print the progress bar.
34 logtostderr: Log messages to stderr instead of stdout.
35 consoletitle: Display progress in console window's titlebar.
36 nopart: Do not use temporary .part files.
37 updatetime: Use the Last-modified header to set output file timestamps.
38 test: Download only first bytes to test the downloader.
39 min_filesize: Skip files smaller than this size
40 max_filesize: Skip files larger than this size
42 Subclasses of this one must re-define the real_download method.
47 def __init__(self
, ydl
, params
):
48 """Create a FileDownloader object with the given options."""
50 self
._progress
_hooks
= []
54 def format_seconds(seconds
):
55 (mins
, secs
) = divmod(seconds
, 60)
56 (hours
, mins
) = divmod(mins
, 60)
60 return '%02d:%02d' % (mins
, secs
)
62 return '%02d:%02d:%02d' % (hours
, mins
, secs
)
65 def calc_percent(byte_counter
, data_len
):
68 return float(byte_counter
) / float(data_len
) * 100.0
71 def format_percent(percent
):
74 return '%6s' % ('%3.1f%%' % percent
)
77 def calc_eta(start
, now
, total
, current
):
81 if current
== 0 or dif
< 0.001: # One millisecond
83 rate
= float(current
) / dif
84 return int((float(total
) - float(current
)) / rate
)
90 return FileDownloader
.format_seconds(eta
)
93 def calc_speed(start
, now
, bytes):
95 if bytes == 0 or dif
< 0.001: # One millisecond
97 return float(bytes) / dif
100 def format_speed(speed
):
102 return '%10s' % '---b/s'
103 return '%10s' % ('%s/s' % format_bytes(speed
))
106 def best_block_size(elapsed_time
, bytes):
107 new_min
= max(bytes / 2.0, 1.0)
108 new_max
= min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
109 if elapsed_time
< 0.001:
111 rate
= bytes / elapsed_time
119 def parse_bytes(bytestr
):
120 """Parse a string indicating a byte quantity into an integer."""
121 matchobj
= re
.match(r
'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr
)
124 number
= float(matchobj
.group(1))
125 multiplier
= 1024.0 ** 'bkmgtpezy'.index(matchobj
.group(2).lower())
126 return int(round(number
* multiplier
))
128 def to_screen(self
, *args
, **kargs
):
129 self
.ydl
.to_screen(*args
, **kargs
)
131 def to_stderr(self
, message
):
132 self
.ydl
.to_screen(message
)
134 def to_console_title(self
, message
):
135 self
.ydl
.to_console_title(message
)
137 def trouble(self
, *args
, **kargs
):
138 self
.ydl
.trouble(*args
, **kargs
)
140 def report_warning(self
, *args
, **kargs
):
141 self
.ydl
.report_warning(*args
, **kargs
)
143 def report_error(self
, *args
, **kargs
):
144 self
.ydl
.report_error(*args
, **kargs
)
146 def slow_down(self
, start_time
, byte_counter
):
147 """Sleep if the download speed is over the rate limit."""
148 rate_limit
= self
.params
.get('ratelimit', None)
149 if rate_limit
is None or byte_counter
== 0:
152 elapsed
= now
- start_time
155 speed
= float(byte_counter
) / elapsed
156 if speed
> rate_limit
:
157 time
.sleep((byte_counter
- rate_limit
* (now
- start_time
)) / rate_limit
)
159 def temp_name(self
, filename
):
160 """Returns a temporary filename for the given filename."""
161 if self
.params
.get('nopart', False) or filename
== u
'-' or \
162 (os
.path
.exists(encodeFilename(filename
)) and not os
.path
.isfile(encodeFilename(filename
))):
164 return filename
+ u
'.part'
166 def undo_temp_name(self
, filename
):
167 if filename
.endswith(u
'.part'):
168 return filename
[:-len(u
'.part')]
171 def try_rename(self
, old_filename
, new_filename
):
173 if old_filename
== new_filename
:
175 os
.rename(encodeFilename(old_filename
), encodeFilename(new_filename
))
176 except (IOError, OSError) as err
:
177 self
.report_error(u
'unable to rename file: %s' % compat_str(err
))
179 def try_utime(self
, filename
, last_modified_hdr
):
180 """Try to set the last-modified time of the given file."""
181 if last_modified_hdr
is None:
183 if not os
.path
.isfile(encodeFilename(filename
)):
185 timestr
= last_modified_hdr
188 filetime
= timeconvert(timestr
)
191 # Ignore obviously invalid dates
195 os
.utime(filename
, (time
.time(), filetime
))
200 def report_destination(self
, filename
):
201 """Report destination filename."""
202 self
.to_screen(u
'[download] Destination: ' + filename
)
204 def _report_progress_status(self
, msg
, is_last_line
=False):
205 fullmsg
= u
'[download] ' + msg
206 if self
.params
.get('progress_with_newline', False):
207 self
.to_screen(fullmsg
)
210 prev_len
= getattr(self
, '_report_progress_prev_line_length',
212 if prev_len
> len(fullmsg
):
213 fullmsg
+= u
' ' * (prev_len
- len(fullmsg
))
214 self
._report
_progress
_prev
_line
_length
= len(fullmsg
)
217 clear_line
= (u
'\r\x1b[K' if sys
.stderr
.isatty() else u
'\r')
218 self
.to_screen(clear_line
+ fullmsg
, skip_eol
=not is_last_line
)
219 self
.to_console_title(u
'youtube-dl ' + msg
)
221 def report_progress(self
, percent
, data_len_str
, speed
, eta
):
222 """Report download progress."""
223 if self
.params
.get('noprogress', False):
226 eta_str
= self
.format_eta(eta
)
228 eta_str
= 'Unknown ETA'
229 if percent
is not None:
230 percent_str
= self
.format_percent(percent
)
232 percent_str
= 'Unknown %'
233 speed_str
= self
.format_speed(speed
)
235 msg
= (u
'%s of %s at %s ETA %s' %
236 (percent_str
, data_len_str
, speed_str
, eta_str
))
237 self
._report
_progress
_status
(msg
)
239 def report_progress_live_stream(self
, downloaded_data_len
, speed
, elapsed
):
240 if self
.params
.get('noprogress', False):
242 downloaded_str
= format_bytes(downloaded_data_len
)
243 speed_str
= self
.format_speed(speed
)
244 elapsed_str
= FileDownloader
.format_seconds(elapsed
)
245 msg
= u
'%s at %s (%s)' % (downloaded_str
, speed_str
, elapsed_str
)
246 self
._report
_progress
_status
(msg
)
248 def report_finish(self
, data_len_str
, tot_time
):
249 """Report download finished."""
250 if self
.params
.get('noprogress', False):
251 self
.to_screen(u
'[download] Download completed')
253 self
._report
_progress
_status
(
254 (u
'100%% of %s in %s' %
255 (data_len_str
, self
.format_seconds(tot_time
))),
258 def report_resuming_byte(self
, resume_len
):
259 """Report attempt to resume at given byte."""
260 self
.to_screen(u
'[download] Resuming download at byte %s' % resume_len
)
262 def report_retry(self
, count
, retries
):
263 """Report retry in case of HTTP error 5xx"""
264 self
.to_screen(u
'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count
, retries
))
266 def report_file_already_downloaded(self
, file_name
):
267 """Report file has already been fully downloaded."""
269 self
.to_screen(u
'[download] %s has already been downloaded' % file_name
)
270 except UnicodeEncodeError:
271 self
.to_screen(u
'[download] The file has already been downloaded')
273 def report_unable_to_resume(self
):
274 """Report it was impossible to resume download."""
275 self
.to_screen(u
'[download] Unable to resume')
277 def download(self
, filename
, info_dict
):
278 """Download to a filename using the info from info_dict
279 Return True on success and False otherwise
281 # Check file already present
282 if self
.params
.get('continuedl', False) and os
.path
.isfile(encodeFilename(filename
)) and not self
.params
.get('nopart', False):
283 self
.report_file_already_downloaded(filename
)
284 self
._hook
_progress
({
285 'filename': filename
,
286 'status': 'finished',
287 'total_bytes': os
.path
.getsize(encodeFilename(filename
)),
291 return self
.real_download(filename
, info_dict
)
293 def real_download(self
, filename
, info_dict
):
294 """Real download process. Redefine in subclasses."""
295 raise NotImplementedError(u
'This method must be implemented by sublcasses')
297 def _hook_progress(self
, status
):
298 for ph
in self
._progress
_hooks
:
301 def add_progress_hook(self
, ph
):
302 """ ph gets called on download progress, with a dictionary with the entries
303 * filename: The final filename
304 * status: One of "downloading" and "finished"
306 It can also have some of the following entries:
308 * downloaded_bytes: Bytes on disks
309 * total_bytes: Total bytes, None if unknown
310 * tmpfilename: The filename we're currently writing to
311 * eta: The estimated time in seconds, None if unknown
312 * speed: The download speed in bytes/second, None if unknown
314 Hooks are guaranteed to be called at least once (with status "finished")
315 if the download is successful.
317 self
._progress
_hooks
.append(ph
)