Merge tag 'upstream/2017.03.26'

[youtubedl] / youtube_dl / downloader / common.py

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 3ae90021a28e661ab532a2d42a7c4e0826d1f46f..2c4470a9549ac0eef1eff8cb413d1e522876e677 100644 (file)
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -4,10 +4,13 @@ import os
 import re
 import sys
 import time
 import re
 import sys
 import time

+import random

 
 

-from ..compat import compat_str
+from ..compat import compat_os_name

 from ..utils import (
     encodeFilename,
 from ..utils import (
     encodeFilename,

+    error_to_compat_str,
+    decodeArgument,

     format_bytes,
     timeconvert,
 )
     format_bytes,
     timeconvert,
 )


@@ -41,7 +44,10 @@ class FileDownloader(object):
     min_filesize:       Skip files smaller than this size
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
     min_filesize:       Skip files smaller than this size
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.

-                        (experimenatal)
+                        (experimental)
+    external_downloader_args:  A list of additional command-line arguments for the
+                        external downloader.
+    hls_use_mpegts:     Use the mpegts container for HLS videos.

 
     Subclasses of this one must re-define the real_download method.
     """
 
     Subclasses of this one must re-define the real_download method.
     """


@@ -110,6 +116,10 @@ class FileDownloader(object):
             return '%10s' % '---b/s'
         return '%10s' % ('%s/s' % format_bytes(speed))
 
             return '%10s' % '---b/s'
         return '%10s' % ('%s/s' % format_bytes(speed))
 

+    @staticmethod
+    def format_retries(retries):
+        return 'inf' if retries == float('inf') else '%.0f' % retries
+

     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)


@@ -153,7 +163,7 @@ class FileDownloader(object):
 
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""
 
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""

-        rate_limit = self.params.get('ratelimit', None)
+        rate_limit = self.params.get('ratelimit')

         if rate_limit is None or byte_counter == 0:
             return
         if now is None:
         if rate_limit is None or byte_counter == 0:
             return
         if now is None:


@@ -183,7 +193,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:

-            self.report_error('unable to rename file: %s' % compat_str(err))
+            self.report_error('unable to rename file: %s' % error_to_compat_str(err))

 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""


@@ -202,7 +212,7 @@ class FileDownloader(object):
             return
         try:
             os.utime(filename, (time.time(), filetime))
             return
         try:
             os.utime(filename, (time.time(), filetime))

-        except:
+        except Exception:

             pass
         return filetime
 
             pass
         return filetime
 


@@ -215,7 +225,7 @@ class FileDownloader(object):
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:

-            if os.name == 'nt':
+            if compat_os_name == 'nt':

                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):
                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):


@@ -292,7 +302,9 @@ class FileDownloader(object):
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""

-        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+        self.to_screen(
+            '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
+            % (count, self.format_retries(retries)))

 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""


@@ -316,13 +328,13 @@ class FileDownloader(object):
         )
 
         continuedl_and_exists = (
         )
 
         continuedl_and_exists = (

-            self.params.get('continuedl', False) and
+            self.params.get('continuedl', True) and

             os.path.isfile(encodeFilename(filename)) and
             not self.params.get('nopart', False)
         )
 
         # Check file already present
             os.path.isfile(encodeFilename(filename)) and
             not self.params.get('nopart', False)
         )
 
         # Check file already present

-        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:
+        if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):

             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,


@@ -331,9 +343,14 @@ class FileDownloader(object):
             })
             return True
 
             })
             return True
 

-        sleep_interval = self.params.get('sleep_interval')
-        if sleep_interval:
-            self.to_screen('[download] Sleeping %s seconds...' % sleep_interval)
+        min_sleep_interval = self.params.get('sleep_interval')
+        if min_sleep_interval:
+            max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
+            sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
+            self.to_screen(
+                '[download] Sleeping %s seconds...' % (
+                    int(sleep_interval) if sleep_interval.is_integer()
+                    else '%.2f' % sleep_interval))

             time.sleep(sleep_interval)
 
         return self.real_download(filename, info_dict)
             time.sleep(sleep_interval)
 
         return self.real_download(filename, info_dict)


@@ -351,19 +368,15 @@ class FileDownloader(object):
         # this interface
         self._progress_hooks.append(ph)
 
         # this interface
         self._progress_hooks.append(ph)
 

-    def _debug_cmd(self, args, subprocess_encoding, exe=None):
+    def _debug_cmd(self, args, exe=None):

         if not self.params.get('verbose', False):
             return
 
         if not self.params.get('verbose', False):
             return
 

+        str_args = [decodeArgument(a) for a in args]
+

         if exe is None:
         if exe is None:

-            exe = os.path.basename(args[0])
+            exe = os.path.basename(str_args[0])

 
 

-        if subprocess_encoding:
-            str_args = [
-                a.decode(subprocess_encoding) if isinstance(a, bytes) else a
-                for a in args]
-        else:
-            str_args = args

         try:
             import pipes
             shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
         try:
             import pipes
             shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))