Tidy the changelog.

[youtubedl] / youtube_dl / downloader / common.py

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index c0af50c59182ad560a23264738d027d4ad3c8697..7bb3a948d2ebd0eaca46ca72f72dfaa2e7ffd1bd 100644 (file)
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -5,8 +5,8 @@ import re
 import sys
 import time
 
 import sys
 import time
 

+from ..compat import compat_str

 from ..utils import (
 from ..utils import (

-    compat_str,

     encodeFilename,
     format_bytes,
     timeconvert,
     encodeFilename,
     format_bytes,
     timeconvert,


@@ -25,21 +25,23 @@ class FileDownloader(object):
 
     Available options:
 
 
     Available options:
 

-    verbose:           Print additional info to stdout.
-    quiet:             Do not print messages to stdout.
-    ratelimit:         Download speed limit, in bytes/sec.
-    retries:           Number of times to retry for HTTP error 5xx
-    buffersize:        Size of download buffer in bytes.
-    noresizebuffer:    Do not automatically resize the download buffer.
-    continuedl:        Try to continue downloads if possible.
-    noprogress:        Do not print the progress bar.
-    logtostderr:       Log messages to stderr instead of stdout.
-    consoletitle:      Display progress in console window's titlebar.
-    nopart:            Do not use temporary .part files.
-    updatetime:        Use the Last-modified header to set output file timestamps.
-    test:              Download only first bytes to test the downloader.
-    min_filesize:      Skip files smaller than this size
-    max_filesize:      Skip files larger than this size
+    verbose:            Print additional info to stdout.
+    quiet:              Do not print messages to stdout.
+    ratelimit:          Download speed limit, in bytes/sec.
+    retries:            Number of times to retry for HTTP error 5xx
+    buffersize:         Size of download buffer in bytes.
+    noresizebuffer:     Do not automatically resize the download buffer.
+    continuedl:         Try to continue downloads if possible.
+    noprogress:         Do not print the progress bar.
+    logtostderr:        Log messages to stderr instead of stdout.
+    consoletitle:       Display progress in console window's titlebar.
+    nopart:             Do not use temporary .part files.
+    updatetime:         Use the Last-modified header to set output file timestamps.
+    test:               Download only first bytes to test the downloader.
+    min_filesize:       Skip files smaller than this size
+    max_filesize:       Skip files larger than this size
+    xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
+                        (experimenatal)

 
     Subclasses of this one must re-define the real_download method.
     """
 
     Subclasses of this one must re-define the real_download method.
     """


@@ -80,6 +82,8 @@ class FileDownloader(object):
     def calc_eta(start, now, total, current):
         if total is None:
             return None
     def calc_eta(start, now, total, current):
         if total is None:
             return None

+        if now is None:
+            now = time.time()

         dif = now - start
         if current == 0 or dif < 0.001:  # One millisecond
             return None
         dif = now - start
         if current == 0 or dif < 0.001:  # One millisecond
             return None


@@ -146,18 +150,19 @@ class FileDownloader(object):
     def report_error(self, *args, **kargs):
         self.ydl.report_error(*args, **kargs)
 
     def report_error(self, *args, **kargs):
         self.ydl.report_error(*args, **kargs)
 

-    def slow_down(self, start_time, byte_counter):
+    def slow_down(self, start_time, now, byte_counter):

         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)
         if rate_limit is None or byte_counter == 0:
             return
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)
         if rate_limit is None or byte_counter == 0:
             return

-        now = time.time()
+        if now is None:
+            now = time.time()

         elapsed = now - start_time
         if elapsed <= 0.0:
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
         elapsed = now - start_time
         if elapsed <= 0.0:
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:

-            time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
+            time.sleep(max((byte_counter // rate_limit) - elapsed, 0))

 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""


@@ -281,8 +286,20 @@ class FileDownloader(object):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
         """
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
         """

+
+        nooverwrites_and_exists = (
+            self.params.get('nooverwrites', False)
+            and os.path.exists(encodeFilename(filename))
+        )
+
+        continuedl_and_exists = (
+            self.params.get('continuedl', False)
+            and os.path.isfile(encodeFilename(filename))
+            and not self.params.get('nopart', False)
+        )
+

         # Check file already present
         # Check file already present

-        if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
+        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:

             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,


@@ -291,6 +308,11 @@ class FileDownloader(object):
             })
             return True
 
             })
             return True
 

+        sleep_interval = self.params.get('sleep_interval')
+        if sleep_interval:
+            self.to_screen('[download] Sleeping %s seconds...' % sleep_interval)
+            time.sleep(sleep_interval)
+

         return self.real_download(filename, info_dict)
 
     def real_download(self, filename, info_dict):
         return self.real_download(filename, info_dict)
 
     def real_download(self, filename, info_dict):


@@ -302,19 +324,27 @@ class FileDownloader(object):
             ph(status)
 
     def add_progress_hook(self, ph):
             ph(status)
 
     def add_progress_hook(self, ph):

-        """ ph gets called on download progress, with a dictionary with the entries
-        * filename: The final filename
-        * status: One of "downloading" and "finished"
+        # See YoutubeDl.py (search for progress_hooks) for a description of
+        # this interface
+        self._progress_hooks.append(ph)

 
 

-        It can also have some of the following entries:
+    def _debug_cmd(self, args, subprocess_encoding, exe=None):
+        if not self.params.get('verbose', False):
+            return

 
 

-        * downloaded_bytes: Bytes on disks
-        * total_bytes: Total bytes, None if unknown
-        * tmpfilename: The filename we're currently writing to
-        * eta: The estimated time in seconds, None if unknown
-        * speed: The download speed in bytes/second, None if unknown
+        if exe is None:
+            exe = os.path.basename(args[0])

 
 

-        Hooks are guaranteed to be called at least once (with status "finished")
-        if the download is successful.
-        """
-        self._progress_hooks.append(ph)
+        if subprocess_encoding:
+            str_args = [
+                a.decode(subprocess_encoding) if isinstance(a, bytes) else a
+                for a in args]
+        else:
+            str_args = args
+        try:
+            import pipes
+            shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
+        except ImportError:
+            shell_quote = repr
+        self.to_screen('[debug] %s command line: %s' % (
+            exe, shell_quote(str_args)))