Merge tag 'upstream/2013.06.21'

[youtubedl] / youtube_dl / FileDownloader.py

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 8a3bdf21b55ac39942cd1c421079750e9c44f9be..f4ce48046f08b21d66fcdecd4e1979cedebfd5dc 100644 (file)
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -322,6 +322,9 @@ class FileDownloader(object):
         filetime = timeconvert(timestr)
         if filetime is None:
             return filetime
         filetime = timeconvert(timestr)
         if filetime is None:
             return filetime

+        # Ignore obviously invalid dates
+        if filetime == 0:
+            return

         try:
             os.utime(filename, (time.time(), filetime))
         except:
         try:
             os.utime(filename, (time.time(), filetime))
         except:


@@ -436,10 +439,11 @@ class FileDownloader(object):
                 return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
         return None
         
                 return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
         return None
         

-    def extract_info(self, url, download=True, ie_key=None):
+    def extract_info(self, url, download=True, ie_key=None, extra_info={}):

         '''
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
         '''
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.

+        extra_info is a dict containing the extra values to add to each result

          '''
         
         if ie_key:
          '''
         
         if ie_key:


@@ -463,10 +467,14 @@ class FileDownloader(object):
                     break
                 if isinstance(ie_result, list):
                     # Backwards compatibility: old IE result format
                     break
                 if isinstance(ie_result, list):
                     # Backwards compatibility: old IE result format

+                    for result in ie_result:
+                        result.update(extra_info)

                     ie_result = {
                         '_type': 'compat_list',
                         'entries': ie_result,
                     }
                     ie_result = {
                         '_type': 'compat_list',
                         'entries': ie_result,
                     }

+                else:
+                    ie_result.update(extra_info)

                 if 'extractor' not in ie_result:
                     ie_result['extractor'] = ie.IE_NAME
                 return self.process_ie_result(ie_result, download=download)
                 if 'extractor' not in ie_result:
                     ie_result['extractor'] = ie.IE_NAME
                 return self.process_ie_result(ie_result, download=download)


@@ -482,7 +490,7 @@ class FileDownloader(object):
         else:
             self.report_error(u'no suitable InfoExtractor: %s' % url)
         
         else:
             self.report_error(u'no suitable InfoExtractor: %s' % url)
         

-    def process_ie_result(self, ie_result, download=True):
+    def process_ie_result(self, ie_result, download=True, extra_info={}):

         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).
         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).


@@ -501,7 +509,12 @@ class FileDownloader(object):
                 self.process_info(ie_result)
             return ie_result
         elif result_type == 'url':
                 self.process_info(ie_result)
             return ie_result
         elif result_type == 'url':

-            return self.extract_info(ie_result['url'], download, ie_key=ie_result.get('ie_key'))
+            # We have to add extra_info to the results because it may be
+            # contained in a playlist
+            return self.extract_info(ie_result['url'],
+                                     download,
+                                     ie_key=ie_result.get('ie_key'),
+                                     extra_info=extra_info)

         elif result_type == 'playlist':
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)
         elif result_type == 'playlist':
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)


@@ -525,9 +538,18 @@ class FileDownloader(object):
 
             for i,entry in enumerate(entries,1):
                 self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
 
             for i,entry in enumerate(entries,1):
                 self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))

-                entry['playlist'] = playlist
-                entry['playlist_index'] = i + playliststart
-                entry_result = self.process_ie_result(entry, download=download)
+                extra = {
+                         'playlist': playlist, 
+                         'playlist_index': i + playliststart,
+                         }
+                if not 'extractor' in entry:
+                    # We set the extractor, if it's an url it will be set then to
+                    # the new extractor, but if it's already a video we must make
+                    # sure it's present: see issue #877
+                    entry['extractor'] = ie_result['extractor']
+                entry_result = self.process_ie_result(entry,
+                                                      download=download,
+                                                      extra_info=extra)

                 playlist_results.append(entry_result)
             ie_result['entries'] = playlist_results
             return ie_result
                 playlist_results.append(entry_result)
             ie_result['entries'] = playlist_results
             return ie_result


@@ -734,7 +756,7 @@ class FileDownloader(object):
             except (IOError, OSError):
                 self.report_warning(u'Unable to remove downloaded video file')
 
             except (IOError, OSError):
                 self.report_warning(u'Unable to remove downloaded video file')
 

-    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):

         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 


@@ -744,18 +766,21 @@ class FileDownloader(object):
         except (OSError, IOError):
             self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
             return False
         except (OSError, IOError):
             self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
             return False

+        verbosity_option = '--verbose' if self.params.get('verbose', False) else '--quiet'

 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
         # the connection was interrumpted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
         # the connection was interrumpted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.

-        basic_args = ['rtmpdump', '-q', '-r', url, '-o', tmpfilename]
+        basic_args = ['rtmpdump', verbosity_option, '-r', url, '-o', tmpfilename]

         if player_url is not None:
         if player_url is not None:

-            basic_args += ['-W', player_url]
+            basic_args += ['--swfVfy', player_url]

         if page_url is not None:
             basic_args += ['--pageUrl', page_url]
         if play_path is not None:
         if page_url is not None:
             basic_args += ['--pageUrl', page_url]
         if play_path is not None:

-            basic_args += ['-y', play_path]
-        args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
+            basic_args += ['--playpath', play_path]
+        if tc_url is not None:
+            basic_args += ['--tcUrl', url]
+        args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]

         if self.params.get('verbose', False):
             try:
                 import pipes
         if self.params.get('verbose', False):
             try:
                 import pipes


@@ -793,6 +818,37 @@ class FileDownloader(object):
             self.report_error(u'rtmpdump exited with code %d' % retval)
             return False
 
             self.report_error(u'rtmpdump exited with code %d' % retval)
             return False
 

+    def _download_with_mplayer(self, filename, url):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url]
+        # Check for mplayer first
+        try:
+            subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+        except (OSError, IOError):
+            self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0] )
+            return False
+
+        # Download using mplayer. 
+        retval = subprocess.call(args)
+        if retval == 0:
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            })
+            return True
+        else:
+            self.to_stderr(u"\n")
+            self.report_error(u'mplayer exited with code %d' % retval)
+            return False
+
+

     def _do_download(self, filename, info_dict):
         url = info_dict['url']
 
     def _do_download(self, filename, info_dict):
         url = info_dict['url']
 


@@ -810,7 +866,12 @@ class FileDownloader(object):
             return self._download_with_rtmpdump(filename, url,
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),
             return self._download_with_rtmpdump(filename, url,
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),

-                                                info_dict.get('play_path', None))
+                                                info_dict.get('play_path', None),
+                                                info_dict.get('tc_url', None))
+
+        # Attempt to download using mplayer
+        if url.startswith('mms') or url.startswith('rtsp'):
+            return self._download_with_mplayer(filename, url)

 
         tmpfilename = self.temp_name(filename)
         stream = None
 
         tmpfilename = self.temp_name(filename)
         stream = None