Merge tag 'upstream/2016.02.22'

[youtubedl] / youtube_dl / downloader / http.py

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 8407727ba43df75a31a53d073cbd430877157b29..f8b69d186ac5ee93c8402f85bc66e7ed59570118 100644 (file)
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -1,15 +1,18 @@
+from __future__ import unicode_literals
+
+import errno

 import os
 import os

+import socket

 import time
 import time

+import re

 
 from .common import FileDownloader
 
 from .common import FileDownloader

+from ..compat import compat_urllib_error

 from ..utils import (
 from ..utils import (

-    compat_urllib_request,
-    compat_urllib_error,

     ContentTooShortError,
     ContentTooShortError,

-

     encodeFilename,
     sanitize_open,
     encodeFilename,
     sanitize_open,

-    format_bytes,
+    sanitized_Request,

 )
 
 
 )
 
 


@@ -21,13 +24,16 @@ class HttpFD(FileDownloader):
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}

-        if 'user_agent' in info_dict:
-            headers['Youtubedl-user-agent'] = info_dict['user_agent']
-        basic_request = compat_urllib_request.Request(url, None, headers)
-        request = compat_urllib_request.Request(url, None, headers)
+        add_headers = info_dict.get('http_headers')
+        if add_headers:
+            headers.update(add_headers)
+        basic_request = sanitized_Request(url, None, headers)
+        request = sanitized_Request(url, None, headers)

 
 

-        if self.params.get('test', False):
-            request.add_header('Range','bytes=0-10240')
+        is_test = self.params.get('test', False)
+
+        if is_test:
+            request.add_header('Range', 'bytes=0-%s' % str(self._TEST_FILE_SIZE - 1))

 
         # Establish possible resume length
         if os.path.isfile(encodeFilename(tmpfilename)):
 
         # Establish possible resume length
         if os.path.isfile(encodeFilename(tmpfilename)):


@@ -37,9 +43,9 @@ class HttpFD(FileDownloader):
 
         open_mode = 'wb'
         if resume_len != 0:
 
         open_mode = 'wb'
         if resume_len != 0:

-            if self.params.get('continuedl', False):
+            if self.params.get('continuedl', True):

                 self.report_resuming_byte(resume_len)
                 self.report_resuming_byte(resume_len)

-                request.add_header('Range','bytes=%d-' % resume_len)
+                request.add_header('Range', 'bytes=%d-' % resume_len)

                 open_mode = 'ab'
             else:
                 resume_len = 0
                 open_mode = 'ab'
             else:
                 resume_len = 0


@@ -49,7 +55,25 @@ class HttpFD(FileDownloader):
         while count <= retries:
             # Establish connection
             try:
         while count <= retries:
             # Establish connection
             try:

-                data = compat_urllib_request.urlopen(request)
+                data = self.ydl.urlopen(request)
+                # When trying to resume, Content-Range HTTP header of response has to be checked
+                # to match the value of requested Range HTTP header. This is due to a webservers
+                # that don't support resuming and serve a whole file with no Content-Range
+                # set in response despite of requested Range (see
+                # https://github.com/rg3/youtube-dl/issues/6057#issuecomment-126129799)
+                if resume_len > 0:
+                    content_range = data.headers.get('Content-Range')
+                    if content_range:
+                        content_range_m = re.search(r'bytes (\d+)-', content_range)
+                        # Content-Range is present and matches requested Range, resume is possible
+                        if content_range_m and resume_len == int(content_range_m.group(1)):
+                            break
+                    # Content-Range is either not present or invalid. Assuming remote webserver is
+                    # trying to send the whole file, resume is not possible, so wiping the local file
+                    # and performing entire redownload
+                    self.report_unable_to_resume()
+                    resume_len = 0
+                    open_mode = 'wb'

                 break
             except (compat_urllib_error.HTTPError, ) as err:
                 if (err.code < 500 or err.code >= 600) and err.code != 416:
                 break
             except (compat_urllib_error.HTTPError, ) as err:
                 if (err.code < 500 or err.code >= 600) and err.code != 416:


@@ -59,7 +83,7 @@ class HttpFD(FileDownloader):
                     # Unable to resume (requested range not satisfiable)
                     try:
                         # Open the connection again without the range header
                     # Unable to resume (requested range not satisfiable)
                     try:
                         # Open the connection again without the range header

-                        data = compat_urllib_request.urlopen(basic_request)
+                        data = self.ydl.urlopen(basic_request)

                         content_length = data.info()['Content-Length']
                     except (compat_urllib_error.HTTPError, ) as err:
                         if err.code < 500 or err.code >= 600:
                         content_length = data.info()['Content-Length']
                     except (compat_urllib_error.HTTPError, ) as err:
                         if err.code < 500 or err.code >= 600:


@@ -80,48 +104,69 @@ class HttpFD(FileDownloader):
                             self._hook_progress({
                                 'filename': filename,
                                 'status': 'finished',
                             self._hook_progress({
                                 'filename': filename,
                                 'status': 'finished',

+                                'downloaded_bytes': resume_len,
+                                'total_bytes': resume_len,

                             })
                             return True
                         else:
                             # The length does not match, we start the download over
                             self.report_unable_to_resume()
                             })
                             return True
                         else:
                             # The length does not match, we start the download over
                             self.report_unable_to_resume()

+                            resume_len = 0

                             open_mode = 'wb'
                             break
                             open_mode = 'wb'
                             break

+            except socket.error as e:
+                if e.errno != errno.ECONNRESET:
+                    # Connection reset is no problem, just retry
+                    raise
+

             # Retry
             count += 1
             if count <= retries:
                 self.report_retry(count, retries)
 
         if count > retries:
             # Retry
             count += 1
             if count <= retries:
                 self.report_retry(count, retries)
 
         if count > retries:

-            self.report_error(u'giving up after %s retries' % retries)
+            self.report_error('giving up after %s retries' % retries)

             return False
 
         data_len = data.info().get('Content-length', None)
             return False
 
         data_len = data.info().get('Content-length', None)

+
+        # Range HTTP header may be ignored/unsupported by a webserver
+        # (e.g. extractor/scivee.py, extractor/bambuser.py).
+        # However, for a test we still would like to download just a piece of a file.
+        # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
+        # block size when downloading a file.
+        if is_test and (data_len is None or int(data_len) > self._TEST_FILE_SIZE):
+            data_len = self._TEST_FILE_SIZE
+

         if data_len is not None:
             data_len = int(data_len) + resume_len
         if data_len is not None:
             data_len = int(data_len) + resume_len

-            min_data_len = self.params.get("min_filesize", None)
-            max_data_len =  self.params.get("max_filesize", None)
+            min_data_len = self.params.get('min_filesize')
+            max_data_len = self.params.get('max_filesize')

             if min_data_len is not None and data_len < min_data_len:
             if min_data_len is not None and data_len < min_data_len:

-                self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
+                self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))

                 return False
             if max_data_len is not None and data_len > max_data_len:
                 return False
             if max_data_len is not None and data_len > max_data_len:

-                self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
+                self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))

                 return False
 
                 return False
 

-        data_len_str = format_bytes(data_len)

         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()

+
+        # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
+        now = None  # needed for slow_down() in the first loop run
+        before = start  # start measuring

         while True:
         while True:

+

             # Download and write
             # Download and write

-            before = time.time()
-            data_block = data.read(block_size)
-            after = time.time()
+            data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
+            byte_counter += len(data_block)
+
+            # exit loop when download is finished

             if len(data_block) == 0:
                 break
             if len(data_block) == 0:
                 break

-            byte_counter += len(data_block)

 
 

-            # Open file just in time
+            # Open destination file just in time

             if stream is None:
                 try:
                     (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
             if stream is None:
                 try:
                     (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)


@@ -129,45 +174,64 @@ class HttpFD(FileDownloader):
                     filename = self.undo_temp_name(tmpfilename)
                     self.report_destination(filename)
                 except (OSError, IOError) as err:
                     filename = self.undo_temp_name(tmpfilename)
                     self.report_destination(filename)
                 except (OSError, IOError) as err:

-                    self.report_error(u'unable to open for writing: %s' % str(err))
+                    self.report_error('unable to open for writing: %s' % str(err))

                     return False
                     return False

+
+                if self.params.get('xattr_set_filesize', False) and data_len is not None:
+                    try:
+                        import xattr
+                        xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
+                    except(OSError, IOError, ImportError) as err:
+                        self.report_error('unable to set filesize xattr: %s' % str(err))
+

             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:

-                self.to_stderr(u"\n")
-                self.report_error(u'unable to write data: %s' % str(err))
+                self.to_stderr('\n')
+                self.report_error('unable to write data: %s' % str(err))

                 return False
                 return False

+
+            # Apply rate limit
+            self.slow_down(start, now, byte_counter - resume_len)
+
+            # end measuring of one loop run
+            now = time.time()
+            after = now
+
+            # Adjust block size

             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
 
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
 

+            before = after
+

             # Progress message
             # Progress message

-            speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
+            speed = self.calc_speed(start, now, byte_counter - resume_len)

             if data_len is None:
             if data_len is None:

-                eta = percent = None
+                eta = None

             else:
             else:

-                percent = self.calc_percent(byte_counter, data_len)

                 eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
                 eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)

-            self.report_progress(percent, data_len_str, speed, eta)

 
             self._hook_progress({
 
             self._hook_progress({

+                'status': 'downloading',

                 'downloaded_bytes': byte_counter,
                 'total_bytes': data_len,
                 'tmpfilename': tmpfilename,
                 'filename': filename,
                 'downloaded_bytes': byte_counter,
                 'total_bytes': data_len,
                 'tmpfilename': tmpfilename,
                 'filename': filename,

-                'status': 'downloading',

                 'eta': eta,
                 'speed': speed,
                 'eta': eta,
                 'speed': speed,

+                'elapsed': now - start,

             })
 
             })
 

-            # Apply rate limit
-            self.slow_down(start, byte_counter - resume_len)
+            if is_test and byte_counter == data_len:
+                break

 
         if stream is None:
 
         if stream is None:

-            self.to_stderr(u"\n")
-            self.report_error(u'Did not get any data blocks')
+            self.to_stderr('\n')
+            self.report_error('Did not get any data blocks')

             return False
             return False

-        stream.close()
-        self.report_finish(data_len_str, (time.time() - start))
+        if tmpfilename != '-':
+            stream.close()
+

         if data_len is not None and byte_counter != data_len:
             raise ContentTooShortError(byte_counter, int(data_len))
         self.try_rename(tmpfilename, filename)
         if data_len is not None and byte_counter != data_len:
             raise ContentTooShortError(byte_counter, int(data_len))
         self.try_rename(tmpfilename, filename)


@@ -181,6 +245,7 @@ class HttpFD(FileDownloader):
             'total_bytes': byte_counter,
             'filename': filename,
             'status': 'finished',
             'total_bytes': byte_counter,
             'filename': filename,
             'status': 'finished',

+            'elapsed': time.time() - start,

         })
 
         return True
         })
 
         return True