]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/downloader/common.py
debian/watch: Make use of the signing key from upstream.
[youtubedl] / youtube_dl / downloader / common.py
1 import os
2 import re
3 import sys
4 import time
5
6 from ..utils import (
7 compat_str,
8 encodeFilename,
9 format_bytes,
10 timeconvert,
11 )
12
13
14 class FileDownloader(object):
15 """File Downloader class.
16
17 File downloader objects are the ones responsible of downloading the
18 actual video file and writing it to disk.
19
20 File downloaders accept a lot of parameters. In order not to saturate
21 the object constructor with arguments, it receives a dictionary of
22 options instead.
23
24 Available options:
25
26 verbose: Print additional info to stdout.
27 quiet: Do not print messages to stdout.
28 ratelimit: Download speed limit, in bytes/sec.
29 retries: Number of times to retry for HTTP error 5xx
30 buffersize: Size of download buffer in bytes.
31 noresizebuffer: Do not automatically resize the download buffer.
32 continuedl: Try to continue downloads if possible.
33 noprogress: Do not print the progress bar.
34 logtostderr: Log messages to stderr instead of stdout.
35 consoletitle: Display progress in console window's titlebar.
36 nopart: Do not use temporary .part files.
37 updatetime: Use the Last-modified header to set output file timestamps.
38 test: Download only first bytes to test the downloader.
39 min_filesize: Skip files smaller than this size
40 max_filesize: Skip files larger than this size
41
42 Subclasses of this one must re-define the real_download method.
43 """
44
45 params = None
46
47 def __init__(self, ydl, params):
48 """Create a FileDownloader object with the given options."""
49 self.ydl = ydl
50 self._progress_hooks = []
51 self.params = params
52
53 @staticmethod
54 def format_seconds(seconds):
55 (mins, secs) = divmod(seconds, 60)
56 (hours, mins) = divmod(mins, 60)
57 if hours > 99:
58 return '--:--:--'
59 if hours == 0:
60 return '%02d:%02d' % (mins, secs)
61 else:
62 return '%02d:%02d:%02d' % (hours, mins, secs)
63
64 @staticmethod
65 def calc_percent(byte_counter, data_len):
66 if data_len is None:
67 return None
68 return float(byte_counter) / float(data_len) * 100.0
69
70 @staticmethod
71 def format_percent(percent):
72 if percent is None:
73 return '---.-%'
74 return '%6s' % ('%3.1f%%' % percent)
75
76 @staticmethod
77 def calc_eta(start, now, total, current):
78 if total is None:
79 return None
80 dif = now - start
81 if current == 0 or dif < 0.001: # One millisecond
82 return None
83 rate = float(current) / dif
84 return int((float(total) - float(current)) / rate)
85
86 @staticmethod
87 def format_eta(eta):
88 if eta is None:
89 return '--:--'
90 return FileDownloader.format_seconds(eta)
91
92 @staticmethod
93 def calc_speed(start, now, bytes):
94 dif = now - start
95 if bytes == 0 or dif < 0.001: # One millisecond
96 return None
97 return float(bytes) / dif
98
99 @staticmethod
100 def format_speed(speed):
101 if speed is None:
102 return '%10s' % '---b/s'
103 return '%10s' % ('%s/s' % format_bytes(speed))
104
105 @staticmethod
106 def best_block_size(elapsed_time, bytes):
107 new_min = max(bytes / 2.0, 1.0)
108 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
109 if elapsed_time < 0.001:
110 return int(new_max)
111 rate = bytes / elapsed_time
112 if rate > new_max:
113 return int(new_max)
114 if rate < new_min:
115 return int(new_min)
116 return int(rate)
117
118 @staticmethod
119 def parse_bytes(bytestr):
120 """Parse a string indicating a byte quantity into an integer."""
121 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
122 if matchobj is None:
123 return None
124 number = float(matchobj.group(1))
125 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
126 return int(round(number * multiplier))
127
128 def to_screen(self, *args, **kargs):
129 self.ydl.to_screen(*args, **kargs)
130
131 def to_stderr(self, message):
132 self.ydl.to_screen(message)
133
134 def to_console_title(self, message):
135 self.ydl.to_console_title(message)
136
137 def trouble(self, *args, **kargs):
138 self.ydl.trouble(*args, **kargs)
139
140 def report_warning(self, *args, **kargs):
141 self.ydl.report_warning(*args, **kargs)
142
143 def report_error(self, *args, **kargs):
144 self.ydl.report_error(*args, **kargs)
145
146 def slow_down(self, start_time, byte_counter):
147 """Sleep if the download speed is over the rate limit."""
148 rate_limit = self.params.get('ratelimit', None)
149 if rate_limit is None or byte_counter == 0:
150 return
151 now = time.time()
152 elapsed = now - start_time
153 if elapsed <= 0.0:
154 return
155 speed = float(byte_counter) / elapsed
156 if speed > rate_limit:
157 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
158
159 def temp_name(self, filename):
160 """Returns a temporary filename for the given filename."""
161 if self.params.get('nopart', False) or filename == u'-' or \
162 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
163 return filename
164 return filename + u'.part'
165
166 def undo_temp_name(self, filename):
167 if filename.endswith(u'.part'):
168 return filename[:-len(u'.part')]
169 return filename
170
171 def try_rename(self, old_filename, new_filename):
172 try:
173 if old_filename == new_filename:
174 return
175 os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
176 except (IOError, OSError) as err:
177 self.report_error(u'unable to rename file: %s' % compat_str(err))
178
179 def try_utime(self, filename, last_modified_hdr):
180 """Try to set the last-modified time of the given file."""
181 if last_modified_hdr is None:
182 return
183 if not os.path.isfile(encodeFilename(filename)):
184 return
185 timestr = last_modified_hdr
186 if timestr is None:
187 return
188 filetime = timeconvert(timestr)
189 if filetime is None:
190 return filetime
191 # Ignore obviously invalid dates
192 if filetime == 0:
193 return
194 try:
195 os.utime(filename, (time.time(), filetime))
196 except:
197 pass
198 return filetime
199
200 def report_destination(self, filename):
201 """Report destination filename."""
202 self.to_screen(u'[download] Destination: ' + filename)
203
204 def _report_progress_status(self, msg, is_last_line=False):
205 fullmsg = u'[download] ' + msg
206 if self.params.get('progress_with_newline', False):
207 self.to_screen(fullmsg)
208 else:
209 if os.name == 'nt':
210 prev_len = getattr(self, '_report_progress_prev_line_length',
211 0)
212 if prev_len > len(fullmsg):
213 fullmsg += u' ' * (prev_len - len(fullmsg))
214 self._report_progress_prev_line_length = len(fullmsg)
215 clear_line = u'\r'
216 else:
217 clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r')
218 self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
219 self.to_console_title(u'youtube-dl ' + msg)
220
221 def report_progress(self, percent, data_len_str, speed, eta):
222 """Report download progress."""
223 if self.params.get('noprogress', False):
224 return
225 if eta is not None:
226 eta_str = self.format_eta(eta)
227 else:
228 eta_str = 'Unknown ETA'
229 if percent is not None:
230 percent_str = self.format_percent(percent)
231 else:
232 percent_str = 'Unknown %'
233 speed_str = self.format_speed(speed)
234
235 msg = (u'%s of %s at %s ETA %s' %
236 (percent_str, data_len_str, speed_str, eta_str))
237 self._report_progress_status(msg)
238
239 def report_progress_live_stream(self, downloaded_data_len, speed, elapsed):
240 if self.params.get('noprogress', False):
241 return
242 downloaded_str = format_bytes(downloaded_data_len)
243 speed_str = self.format_speed(speed)
244 elapsed_str = FileDownloader.format_seconds(elapsed)
245 msg = u'%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
246 self._report_progress_status(msg)
247
248 def report_finish(self, data_len_str, tot_time):
249 """Report download finished."""
250 if self.params.get('noprogress', False):
251 self.to_screen(u'[download] Download completed')
252 else:
253 self._report_progress_status(
254 (u'100%% of %s in %s' %
255 (data_len_str, self.format_seconds(tot_time))),
256 is_last_line=True)
257
258 def report_resuming_byte(self, resume_len):
259 """Report attempt to resume at given byte."""
260 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
261
262 def report_retry(self, count, retries):
263 """Report retry in case of HTTP error 5xx"""
264 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
265
266 def report_file_already_downloaded(self, file_name):
267 """Report file has already been fully downloaded."""
268 try:
269 self.to_screen(u'[download] %s has already been downloaded' % file_name)
270 except UnicodeEncodeError:
271 self.to_screen(u'[download] The file has already been downloaded')
272
273 def report_unable_to_resume(self):
274 """Report it was impossible to resume download."""
275 self.to_screen(u'[download] Unable to resume')
276
277 def download(self, filename, info_dict):
278 """Download to a filename using the info from info_dict
279 Return True on success and False otherwise
280 """
281 # Check file already present
282 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
283 self.report_file_already_downloaded(filename)
284 self._hook_progress({
285 'filename': filename,
286 'status': 'finished',
287 'total_bytes': os.path.getsize(encodeFilename(filename)),
288 })
289 return True
290
291 return self.real_download(filename, info_dict)
292
293 def real_download(self, filename, info_dict):
294 """Real download process. Redefine in subclasses."""
295 raise NotImplementedError(u'This method must be implemented by sublcasses')
296
297 def _hook_progress(self, status):
298 for ph in self._progress_hooks:
299 ph(status)
300
301 def add_progress_hook(self, ph):
302 """ ph gets called on download progress, with a dictionary with the entries
303 * filename: The final filename
304 * status: One of "downloading" and "finished"
305
306 It can also have some of the following entries:
307
308 * downloaded_bytes: Bytes on disks
309 * total_bytes: Total bytes, None if unknown
310 * tmpfilename: The filename we're currently writing to
311 * eta: The estimated time in seconds, None if unknown
312 * speed: The download speed in bytes/second, None if unknown
313
314 Hooks are guaranteed to be called at least once (with status "finished")
315 if the download is successful.
316 """
317 self._progress_hooks.append(ph)