Merge tag 'upstream/2017.03.26'

[youtubedl] / youtube_dl / downloader / common.py

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 97e755d4baa56972a9a4e5223a6871edd8bf0565..2c4470a9549ac0eef1eff8cb413d1e522876e677 100644 (file)
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -4,10 +4,12 @@ import os
 import re
 import sys
 import time
 import re
 import sys
 import time

+import random

 
 

-from ..compat import compat_str
+from ..compat import compat_os_name

 from ..utils import (
     encodeFilename,
 from ..utils import (
     encodeFilename,

+    error_to_compat_str,

     decodeArgument,
     format_bytes,
     timeconvert,
     decodeArgument,
     format_bytes,
     timeconvert,


@@ -42,9 +44,10 @@ class FileDownloader(object):
     min_filesize:       Skip files smaller than this size
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
     min_filesize:       Skip files smaller than this size
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.

-                        (experimenatal)
+                        (experimental)

     external_downloader_args:  A list of additional command-line arguments for the
                         external downloader.
     external_downloader_args:  A list of additional command-line arguments for the
                         external downloader.

+    hls_use_mpegts:     Use the mpegts container for HLS videos.

 
     Subclasses of this one must re-define the real_download method.
     """
 
     Subclasses of this one must re-define the real_download method.
     """


@@ -113,6 +116,10 @@ class FileDownloader(object):
             return '%10s' % '---b/s'
         return '%10s' % ('%s/s' % format_bytes(speed))
 
             return '%10s' % '---b/s'
         return '%10s' % ('%s/s' % format_bytes(speed))
 

+    @staticmethod
+    def format_retries(retries):
+        return 'inf' if retries == float('inf') else '%.0f' % retries
+

     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)


@@ -156,7 +163,7 @@ class FileDownloader(object):
 
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""
 
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""

-        rate_limit = self.params.get('ratelimit', None)
+        rate_limit = self.params.get('ratelimit')

         if rate_limit is None or byte_counter == 0:
             return
         if now is None:
         if rate_limit is None or byte_counter == 0:
             return
         if now is None:


@@ -186,7 +193,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:

-            self.report_error('unable to rename file: %s' % compat_str(err))
+            self.report_error('unable to rename file: %s' % error_to_compat_str(err))

 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""


@@ -218,7 +225,7 @@ class FileDownloader(object):
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:

-            if os.name == 'nt':
+            if compat_os_name == 'nt':

                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):
                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):


@@ -295,7 +302,9 @@ class FileDownloader(object):
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""

-        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+        self.to_screen(
+            '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
+            % (count, self.format_retries(retries)))

 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""


@@ -325,7 +334,7 @@ class FileDownloader(object):
         )
 
         # Check file already present
         )
 
         # Check file already present

-        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:
+        if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):

             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,


@@ -334,9 +343,14 @@ class FileDownloader(object):
             })
             return True
 
             })
             return True
 

-        sleep_interval = self.params.get('sleep_interval')
-        if sleep_interval:
-            self.to_screen('[download] Sleeping %s seconds...' % sleep_interval)
+        min_sleep_interval = self.params.get('sleep_interval')
+        if min_sleep_interval:
+            max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
+            sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
+            self.to_screen(
+                '[download] Sleeping %s seconds...' % (
+                    int(sleep_interval) if sleep_interval.is_integer()
+                    else '%.2f' % sleep_interval))

             time.sleep(sleep_interval)
 
         return self.real_download(filename, info_dict)
             time.sleep(sleep_interval)
 
         return self.real_download(filename, info_dict)