Update changelog.

[youtubedl] / youtube_dl / FileDownloader.py

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 7c5ac4bc2ecae6d3440266a98b2034ac5f56867a..088f595866372e360e425a3aeee196374f504b7c 100644 (file)
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -4,12 +4,19 @@ import re
 import subprocess
 import sys
 import time
 import subprocess
 import sys
 import time

-import traceback

 
 if os.name == 'nt':
     import ctypes
 
 
 if os.name == 'nt':
     import ctypes
 

-from .utils import *
+from .utils import (
+    compat_urllib_error,
+    compat_urllib_request,
+    ContentTooShortError,
+    determine_ext,
+    encodeFilename,
+    sanitize_open,
+    timeconvert,
+)

 
 
 class FileDownloader(object):
 
 
 class FileDownloader(object):


@@ -66,7 +73,7 @@ class FileDownloader(object):
     @staticmethod
     def format_seconds(seconds):
         (mins, secs) = divmod(seconds, 60)
     @staticmethod
     def format_seconds(seconds):
         (mins, secs) = divmod(seconds, 60)

-        (hours, eta_mins) = divmod(mins, 60)
+        (hours, mins) = divmod(mins, 60)

         if hours > 99:
             return '--:--:--'
         if hours == 0:
         if hours > 99:
             return '--:--:--'
         if hours == 0:


@@ -77,26 +84,43 @@ class FileDownloader(object):
     @staticmethod
     def calc_percent(byte_counter, data_len):
         if data_len is None:
     @staticmethod
     def calc_percent(byte_counter, data_len):
         if data_len is None:

+            return None
+        return float(byte_counter) / float(data_len) * 100.0
+
+    @staticmethod
+    def format_percent(percent):
+        if percent is None:

             return '---.-%'
             return '---.-%'

-        return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
+        return '%6s' % ('%3.1f%%' % percent)

 
     @staticmethod
     def calc_eta(start, now, total, current):
         if total is None:
 
     @staticmethod
     def calc_eta(start, now, total, current):
         if total is None:

-            return '--:--'
+            return None

         dif = now - start
         if current == 0 or dif < 0.001: # One millisecond
         dif = now - start
         if current == 0 or dif < 0.001: # One millisecond

-            return '--:--'
+            return None

         rate = float(current) / dif
         rate = float(current) / dif

-        eta = int((float(total) - float(current)) / rate)
+        return int((float(total) - float(current)) / rate)
+
+    @staticmethod
+    def format_eta(eta):
+        if eta is None:
+            return '--:--'

         return FileDownloader.format_seconds(eta)
 
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
         if bytes == 0 or dif < 0.001: # One millisecond
         return FileDownloader.format_seconds(eta)
 
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
         if bytes == 0 or dif < 0.001: # One millisecond

+            return None
+        return float(bytes) / dif
+
+    @staticmethod
+    def format_speed(speed):
+        if speed is None:

             return '%10s' % '---b/s'
             return '%10s' % '---b/s'

-        return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
+        return '%10s' % ('%s/s' % FileDownloader.format_bytes(speed))

 
     @staticmethod
     def best_block_size(elapsed_time, bytes):
 
     @staticmethod
     def best_block_size(elapsed_time, bytes):


@@ -177,7 +201,7 @@ class FileDownloader(object):
             if old_filename == new_filename:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
             if old_filename == new_filename:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))

-        except (IOError, OSError) as err:
+        except (IOError, OSError):

             self.report_error(u'unable to rename file')
 
     def try_utime(self, filename, last_modified_hdr):
             self.report_error(u'unable to rename file')
 
     def try_utime(self, filename, last_modified_hdr):


@@ -205,11 +229,20 @@ class FileDownloader(object):
         """Report destination filename."""
         self.to_screen(u'[download] Destination: ' + filename)
 
         """Report destination filename."""
         self.to_screen(u'[download] Destination: ' + filename)
 

-    def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
+    def report_progress(self, percent, data_len_str, speed, eta):

         """Report download progress."""
         if self.params.get('noprogress', False):
             return
         clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
         """Report download progress."""
         if self.params.get('noprogress', False):
             return
         clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')

+        if eta is not None:
+            eta_str = self.format_eta(eta)
+        else:
+            eta_str = 'Unknown ETA'
+        if percent is not None:
+            percent_str = self.format_percent(percent)
+        else:
+            percent_str = 'Unknown %'
+        speed_str = self.format_speed(speed)

         if self.params.get('progress_with_newline', False):
             self.to_screen(u'[download] %s of %s at %s ETA %s' %
                 (percent_str, data_len_str, speed_str, eta_str))
         if self.params.get('progress_with_newline', False):
             self.to_screen(u'[download] %s of %s at %s ETA %s' %
                 (percent_str, data_len_str, speed_str, eta_str))


@@ -231,7 +264,7 @@ class FileDownloader(object):
         """Report file has already been fully downloaded."""
         try:
             self.to_screen(u'[download] %s has already been downloaded' % file_name)
         """Report file has already been fully downloaded."""
         try:
             self.to_screen(u'[download] %s has already been downloaded' % file_name)

-        except (UnicodeEncodeError) as err:
+        except UnicodeEncodeError:

             self.to_screen(u'[download] The file has already been downloaded')
 
     def report_unable_to_resume(self):
             self.to_screen(u'[download] The file has already been downloaded')
 
     def report_unable_to_resume(self):


@@ -247,9 +280,10 @@ class FileDownloader(object):
             self.to_screen(u'\r%s[download] 100%% of %s in %s' %
                 (clear_line, data_len_str, self.format_seconds(tot_time)))
 
             self.to_screen(u'\r%s[download] 100%% of %s in %s' %
                 (clear_line, data_len_str, self.format_seconds(tot_time)))
 

-    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live):

         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)

+        test = self.params.get('test', False)

 
         # Check for rtmpdump first
         try:
 
         # Check for rtmpdump first
         try:


@@ -271,6 +305,10 @@ class FileDownloader(object):
             basic_args += ['--playpath', play_path]
         if tc_url is not None:
             basic_args += ['--tcUrl', url]
             basic_args += ['--playpath', play_path]
         if tc_url is not None:
             basic_args += ['--tcUrl', url]

+        if test:
+            basic_args += ['--stop', '1']
+        if live:
+            basic_args += ['--live']

         args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:
         args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:


@@ -280,7 +318,7 @@ class FileDownloader(object):
                 shell_quote = repr
             self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
         retval = subprocess.call(args)
                 shell_quote = repr
             self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
         retval = subprocess.call(args)

-        while retval == 2 or retval == 1:
+        while (retval == 2 or retval == 1) and not test:

             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
             time.sleep(5.0) # This seems to be needed
             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
             time.sleep(5.0) # This seems to be needed


@@ -293,7 +331,7 @@ class FileDownloader(object):
                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
                 retval = 0
                 break
                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
                 retval = 0
                 break

-        if retval == 0:
+        if retval == 0 or (test and retval == 2):

             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
             self.try_rename(tmpfilename, filename)
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
             self.try_rename(tmpfilename, filename)


@@ -343,15 +381,20 @@ class FileDownloader(object):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 

-        args = ['ffmpeg', '-y', '-i', url, '-f', 'mp4', tmpfilename]
-        # Check for ffmpeg first
-        try:
-            subprocess.call(['ffmpeg', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
-        except (OSError, IOError):
-            self.report_error(u'm3u8 download detected but "%s" could not be run' % args[0] )
-            return False
+        args = ['-y', '-i', url, '-f', 'mp4', '-c', 'copy',
+            '-bsf:a', 'aac_adtstoasc', tmpfilename]

 
 

-        retval = subprocess.call(args)
+        for program in ['avconv', 'ffmpeg']:
+            try:
+                subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+                break
+            except (OSError, IOError):
+                pass
+        else:
+            self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found')
+        cmd = [program] + args
+
+        retval = subprocess.call(cmd)

         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))


@@ -378,6 +421,7 @@ class FileDownloader(object):
             self._hook_progress({
                 'filename': filename,
                 'status': 'finished',
             self._hook_progress({
                 'filename': filename,
                 'status': 'finished',

+                'total_bytes': os.path.getsize(encodeFilename(filename)),

             })
             return True
 
             })
             return True
 


@@ -387,7 +431,8 @@ class FileDownloader(object):
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),
                                                 info_dict.get('play_path', None),
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),
                                                 info_dict.get('play_path', None),

-                                                info_dict.get('tc_url', None))
+                                                info_dict.get('tc_url', None),
+                                                info_dict.get('rtmp_live', False))

 
         # Attempt to download using mplayer
         if url.startswith('mms') or url.startswith('rtsp'):
 
         # Attempt to download using mplayer
         if url.startswith('mms') or url.startswith('rtsp'):


@@ -524,13 +569,13 @@ class FileDownloader(object):
                 block_size = self.best_block_size(after - before, len(data_block))
 
             # Progress message
                 block_size = self.best_block_size(after - before, len(data_block))
 
             # Progress message

-            speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
+            speed = self.calc_speed(start, time.time(), byte_counter - resume_len)

             if data_len is None:
             if data_len is None:

-                self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
+                eta = percent = None

             else:
             else:

-                percent_str = self.calc_percent(byte_counter, data_len)
-                eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
-                self.report_progress(percent_str, data_len_str, speed_str, eta_str)
+                percent = self.calc_percent(byte_counter, data_len)
+                eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
+            self.report_progress(percent, data_len_str, speed, eta)

 
             self._hook_progress({
                 'downloaded_bytes': byte_counter,
 
             self._hook_progress({
                 'downloaded_bytes': byte_counter,


@@ -538,6 +583,8 @@ class FileDownloader(object):
                 'tmpfilename': tmpfilename,
                 'filename': filename,
                 'status': 'downloading',
                 'tmpfilename': tmpfilename,
                 'filename': filename,
                 'status': 'downloading',

+                'eta': eta,
+                'speed': speed,

             })
 
             # Apply rate limit
             })
 
             # Apply rate limit


@@ -580,6 +627,8 @@ class FileDownloader(object):
         * downloaded_bytes: Bytes on disks
         * total_bytes: Total bytes, None if unknown
         * tmpfilename: The filename we're currently writing to
         * downloaded_bytes: Bytes on disks
         * total_bytes: Total bytes, None if unknown
         * tmpfilename: The filename we're currently writing to

+        * eta: The estimated time in seconds, None if unknown
+        * speed: The download speed in bytes/second, None if unknown

 
         Hooks are guaranteed to be called at least once (with status "finished")
         if the download is successful.
 
         Hooks are guaranteed to be called at least once (with status "finished")
         if the download is successful.