Initiate a new release.

[youtubedl] / youtube_dl / downloader / http.py

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 8491cee8aa2769e8465176411a92ded85b07ad13..49170cf9d47634602efe7832b235e4a751e25817 100644 (file)
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -3,12 +3,16 @@ from __future__ import unicode_literals
 import os
 import time
 
 import os
 import time
 

+from socket import error as SocketError
+import errno
+

 from .common import FileDownloader
 from .common import FileDownloader

-from ..utils import (
+from ..compat import (

     compat_urllib_request,
     compat_urllib_error,
     compat_urllib_request,
     compat_urllib_error,

+)
+from ..utils import (

     ContentTooShortError,
     ContentTooShortError,

-

     encodeFilename,
     sanitize_open,
     format_bytes,
     encodeFilename,
     sanitize_open,
     format_bytes,


@@ -23,10 +27,6 @@ class HttpFD(FileDownloader):
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}

-        if 'user_agent' in info_dict:
-            headers['Youtubedl-user-agent'] = info_dict['user_agent']
-        if 'http_referer' in info_dict:
-            headers['Referer'] = info_dict['http_referer']

         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)


@@ -102,6 +102,11 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
                             resume_len = 0
                             open_mode = 'wb'
                             break

+            except SocketError as e:
+                if e.errno != errno.ECONNRESET:
+                    # Connection reset is no problem, just retry
+                    raise
+

             # Retry
             count += 1
             if count <= retries:
             # Retry
             count += 1
             if count <= retries:


@@ -136,16 +141,21 @@ class HttpFD(FileDownloader):
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()

+
+        # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
+        now = None  # needed for slow_down() in the first loop run
+        before = start  # start measuring

         while True:
         while True:

+

             # Download and write
             # Download and write

-            before = time.time()

             data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
             data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))

-            after = time.time()
+            byte_counter += len(data_block)
+
+            # exit loop when download is finished

             if len(data_block) == 0:
                 break
             if len(data_block) == 0:
                 break

-            byte_counter += len(data_block)

 
 

-            # Open file just in time
+            # Open destination file just in time

             if stream is None:
                 try:
                     (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
             if stream is None:
                 try:
                     (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)


@@ -155,17 +165,36 @@ class HttpFD(FileDownloader):
                 except (OSError, IOError) as err:
                     self.report_error('unable to open for writing: %s' % str(err))
                     return False
                 except (OSError, IOError) as err:
                     self.report_error('unable to open for writing: %s' % str(err))
                     return False

+
+                if self.params.get('xattr_set_filesize', False) and data_len is not None:
+                    try:
+                        import xattr
+                        xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
+                    except(OSError, IOError, ImportError) as err:
+                        self.report_error('unable to set filesize xattr: %s' % str(err))
+

             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
                 self.to_stderr('\n')
                 self.report_error('unable to write data: %s' % str(err))
                 return False
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
                 self.to_stderr('\n')
                 self.report_error('unable to write data: %s' % str(err))
                 return False

+
+            # Apply rate limit
+            self.slow_down(start, now, byte_counter - resume_len)
+
+            # end measuring of one loop run
+            now = time.time()
+            after = now
+
+            # Adjust block size

             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
 
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
 

+            before = after
+

             # Progress message
             # Progress message

-            speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
+            speed = self.calc_speed(start, now, byte_counter - resume_len)

             if data_len is None:
                 eta = percent = None
             else:
             if data_len is None:
                 eta = percent = None
             else:


@@ -186,9 +215,6 @@ class HttpFD(FileDownloader):
             if is_test and byte_counter == data_len:
                 break
 
             if is_test and byte_counter == data_len:
                 break
 

-            # Apply rate limit
-            self.slow_down(start, byte_counter - resume_len)
-

         if stream is None:
             self.to_stderr('\n')
             self.report_error('Did not get any data blocks')
         if stream is None:
             self.to_stderr('\n')
             self.report_error('Did not get any data blocks')