Update upstream source from tag 'upstream/2019.09.01'

[youtubedl] / youtube_dl / downloader / common.py

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 3ae90021a28e661ab532a2d42a7c4e0826d1f46f..1cdba89cd9b093c1cc45071d34e307028aac02ea 100644 (file)
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -4,11 +4,15 @@ import os
 import re
 import sys
 import time
 import re
 import sys
 import time

+import random

 
 

-from ..compat import compat_str
+from ..compat import compat_os_name

 from ..utils import (
 from ..utils import (

+    decodeArgument,

     encodeFilename,
     encodeFilename,

+    error_to_compat_str,

     format_bytes,
     format_bytes,

+    shell_quote,

     timeconvert,
 )
 
     timeconvert,
 )
 


@@ -41,7 +45,12 @@ class FileDownloader(object):
     min_filesize:       Skip files smaller than this size
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
     min_filesize:       Skip files smaller than this size
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.

-                        (experimenatal)
+    external_downloader_args:  A list of additional command-line arguments for the
+                        external downloader.
+    hls_use_mpegts:     Use the mpegts container for HLS videos.
+    http_chunk_size:    Size of a chunk for chunk-based HTTP downloading. May be
+                        useful for bypassing bandwidth throttling imposed by
+                        a webserver (experimental)

 
     Subclasses of this one must re-define the real_download method.
     """
 
     Subclasses of this one must re-define the real_download method.
     """


@@ -110,6 +119,10 @@ class FileDownloader(object):
             return '%10s' % '---b/s'
         return '%10s' % ('%s/s' % format_bytes(speed))
 
             return '%10s' % '---b/s'
         return '%10s' % ('%s/s' % format_bytes(speed))
 

+    @staticmethod
+    def format_retries(retries):
+        return 'inf' if retries == float('inf') else '%.0f' % retries
+

     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)


@@ -153,7 +166,7 @@ class FileDownloader(object):
 
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""
 
     def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""

-        rate_limit = self.params.get('ratelimit', None)
+        rate_limit = self.params.get('ratelimit')

         if rate_limit is None or byte_counter == 0:
             return
         if now is None:
         if rate_limit is None or byte_counter == 0:
             return
         if now is None:


@@ -163,7 +176,9 @@ class FileDownloader(object):
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:

-            time.sleep(max((byte_counter // rate_limit) - elapsed, 0))
+            sleep_time = float(byte_counter) / rate_limit - elapsed
+            if sleep_time > 0:
+                time.sleep(sleep_time)

 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""


@@ -177,13 +192,16 @@ class FileDownloader(object):
             return filename[:-len('.part')]
         return filename
 
             return filename[:-len('.part')]
         return filename
 

+    def ytdl_filename(self, filename):
+        return filename + '.ytdl'
+

     def try_rename(self, old_filename, new_filename):
         try:
             if old_filename == new_filename:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
     def try_rename(self, old_filename, new_filename):
         try:
             if old_filename == new_filename:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:

-            self.report_error('unable to rename file: %s' % compat_str(err))
+            self.report_error('unable to rename file: %s' % error_to_compat_str(err))

 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""


@@ -202,7 +220,7 @@ class FileDownloader(object):
             return
         try:
             os.utime(filename, (time.time(), filetime))
             return
         try:
             os.utime(filename, (time.time(), filetime))

-        except:
+        except Exception:

             pass
         return filetime
 
             pass
         return filetime
 


@@ -215,7 +233,7 @@ class FileDownloader(object):
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:

-            if os.name == 'nt':
+            if compat_os_name == 'nt':

                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):
                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):


@@ -232,12 +250,13 @@ class FileDownloader(object):
             if self.params.get('noprogress', False):
                 self.to_screen('[download] Download completed')
             else:
             if self.params.get('noprogress', False):
                 self.to_screen('[download] Download completed')
             else:

-                s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+                msg_template = '100%%'
+                if s.get('total_bytes') is not None:
+                    s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+                    msg_template += ' of %(_total_bytes_str)s'

                 if s.get('elapsed') is not None:
                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])
                 if s.get('elapsed') is not None:
                     s['_elapsed_str'] = self.format_seconds(s['elapsed'])

-                    msg_template = '100%% of %(_total_bytes_str)s in %(_elapsed_str)s'
-                else:
-                    msg_template = '100%% of %(_total_bytes_str)s'
+                    msg_template += ' in %(_elapsed_str)s'

                 self._report_progress_status(
                     msg_template % s, is_last_line=True)
 
                 self._report_progress_status(
                     msg_template % s, is_last_line=True)
 


@@ -290,9 +309,11 @@ class FileDownloader(object):
         """Report attempt to resume at given byte."""
         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 
         """Report attempt to resume at given byte."""
         self.to_screen('[download] Resuming download at byte %s' % resume_len)
 

-    def report_retry(self, count, retries):
+    def report_retry(self, err, count, retries):

         """Report retry in case of HTTP error 5xx"""
         """Report retry in case of HTTP error 5xx"""

-        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+        self.to_screen(
+            '[download] Got server HTTP error: %s. Retrying (attempt %d of %s)...'
+            % (error_to_compat_str(err), count, self.format_retries(retries)))

 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""


@@ -311,29 +332,35 @@ class FileDownloader(object):
         """
 
         nooverwrites_and_exists = (
         """
 
         nooverwrites_and_exists = (

-            self.params.get('nooverwrites', False) and
-            os.path.exists(encodeFilename(filename))
+            self.params.get('nooverwrites', False)
+            and os.path.exists(encodeFilename(filename))

         )
 
         )
 

-        continuedl_and_exists = (
-            self.params.get('continuedl', False) and
-            os.path.isfile(encodeFilename(filename)) and
-            not self.params.get('nopart', False)
-        )
-
-        # Check file already present
-        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:
-            self.report_file_already_downloaded(filename)
-            self._hook_progress({
-                'filename': filename,
-                'status': 'finished',
-                'total_bytes': os.path.getsize(encodeFilename(filename)),
-            })
-            return True
-
-        sleep_interval = self.params.get('sleep_interval')
-        if sleep_interval:
-            self.to_screen('[download] Sleeping %s seconds...' % sleep_interval)
+        if not hasattr(filename, 'write'):
+            continuedl_and_exists = (
+                self.params.get('continuedl', True)
+                and os.path.isfile(encodeFilename(filename))
+                and not self.params.get('nopart', False)
+            )
+
+            # Check file already present
+            if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
+                self.report_file_already_downloaded(filename)
+                self._hook_progress({
+                    'filename': filename,
+                    'status': 'finished',
+                    'total_bytes': os.path.getsize(encodeFilename(filename)),
+                })
+                return True
+
+        min_sleep_interval = self.params.get('sleep_interval')
+        if min_sleep_interval:
+            max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
+            sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
+            self.to_screen(
+                '[download] Sleeping %s seconds...' % (
+                    int(sleep_interval) if sleep_interval.is_integer()
+                    else '%.2f' % sleep_interval))

             time.sleep(sleep_interval)
 
         return self.real_download(filename, info_dict)
             time.sleep(sleep_interval)
 
         return self.real_download(filename, info_dict)


@@ -351,23 +378,14 @@ class FileDownloader(object):
         # this interface
         self._progress_hooks.append(ph)
 
         # this interface
         self._progress_hooks.append(ph)
 

-    def _debug_cmd(self, args, subprocess_encoding, exe=None):
+    def _debug_cmd(self, args, exe=None):

         if not self.params.get('verbose', False):
             return
 
         if not self.params.get('verbose', False):
             return
 

+        str_args = [decodeArgument(a) for a in args]
+

         if exe is None:
         if exe is None:

-            exe = os.path.basename(args[0])
+            exe = os.path.basename(str_args[0])

 
 

-        if subprocess_encoding:
-            str_args = [
-                a.decode(subprocess_encoding) if isinstance(a, bytes) else a
-                for a in args]
-        else:
-            str_args = args
-        try:
-            import pipes
-            shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
-        except ImportError:
-            shell_quote = repr

         self.to_screen('[debug] %s command line: %s' % (
             exe, shell_quote(str_args)))
         self.to_screen('[debug] %s command line: %s' % (
             exe, shell_quote(str_args)))