Merge tag 'upstream/2014.12.01'

[youtubedl] / youtube_dl / YoutubeDL.py

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 73a372df4724e05607acd0fb202ad7e774f680cc..21c7c298a830dd8b4a1b4651419506aaa8c8a8bc 100755 (executable)
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -22,13 +22,15 @@ import traceback
 if os.name == 'nt':
     import ctypes
 
 if os.name == 'nt':
     import ctypes
 

-from .utils import (
+from .compat import (

     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
     compat_str,
     compat_urllib_error,
     compat_urllib_request,
     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
     compat_str,
     compat_urllib_error,
     compat_urllib_request,

+)
+from .utils import (

     escape_url,
     ContentTooShortError,
     date_from_str,
     escape_url,
     ContentTooShortError,
     date_from_str,


@@ -58,10 +60,12 @@ from .utils import (
     write_string,
     YoutubeDLHandler,
     prepend_extension,
     write_string,
     YoutubeDLHandler,
     prepend_extension,

+    args_to_str,

 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader

+from .downloader.rtmp import rtmpdump_version

 from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
 from .version import __version__
 
 from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
 from .version import __version__
 


@@ -250,6 +254,22 @@ class YoutubeDL(object):
             self.print_debug_header()
             self.add_default_info_extractors()
 
             self.print_debug_header()
             self.add_default_info_extractors()
 

+    def warn_if_short_id(self, argv):
+        # short YouTube ID starting with dash?
+        idxs = [
+            i for i, a in enumerate(argv)
+            if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
+        if idxs:
+            correct_argv = (
+                ['youtube-dl'] +
+                [a for i, a in enumerate(argv) if i not in idxs] +
+                ['--'] + [argv[i] for i in idxs]
+            )
+            self.report_warning(
+                'Long argument string detected. '
+                'Use -- to separate parameters and URLs, like this:\n%s\n' %
+                args_to_str(correct_argv))
+

     def add_info_extractor(self, ie):
         """Add an InfoExtractor object to the end of the list."""
         self._ies.append(ie)
     def add_info_extractor(self, ie):
         """Add an InfoExtractor object to the end of the list."""
         self._ies.append(ie)


@@ -294,7 +314,7 @@ class YoutubeDL(object):
         self._output_process.stdin.write((message + '\n').encode('utf-8'))
         self._output_process.stdin.flush()
         res = ''.join(self._output_channel.readline().decode('utf-8')
         self._output_process.stdin.write((message + '\n').encode('utf-8'))
         self._output_process.stdin.flush()
         res = ''.join(self._output_channel.readline().decode('utf-8')

-                       for _ in range(line_count))
+                      for _ in range(line_count))

         return res[:-len('\n')]
 
     def to_screen(self, message, skip_eol=False):
         return res[:-len('\n')]
 
     def to_screen(self, message, skip_eol=False):


@@ -531,7 +551,7 @@ class YoutubeDL(object):
 
             try:
                 ie_result = ie.extract(url)
 
             try:
                 ie_result = ie.extract(url)

-                if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
+                if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)

                     break
                 if isinstance(ie_result, list):
                     # Backwards compatibility: old IE result format
                     break
                 if isinstance(ie_result, list):
                     # Backwards compatibility: old IE result format


@@ -544,7 +564,7 @@ class YoutubeDL(object):
                     return self.process_ie_result(ie_result, download, extra_info)
                 else:
                     return ie_result
                     return self.process_ie_result(ie_result, download, extra_info)
                 else:
                     return ie_result

-            except ExtractorError as de: # An error we somewhat expected
+            except ExtractorError as de:  # An error we somewhat expected

                 self.report_error(compat_str(de), de.format_traceback())
                 break
             except MaxDownloadsReached:
                 self.report_error(compat_str(de), de.format_traceback())
                 break
             except MaxDownloadsReached:


@@ -621,7 +641,7 @@ class YoutubeDL(object):
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)

-        elif result_type == 'playlist':
+        elif result_type == 'playlist' or result_type == 'multi_video':

             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)
             self.to_screen('[download] Downloading playlist: %s' % playlist)
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)
             self.to_screen('[download] Downloading playlist: %s' % playlist)


@@ -655,6 +675,8 @@ class YoutubeDL(object):
                 extra = {
                     'n_entries': n_entries,
                     'playlist': playlist,
                 extra = {
                     'n_entries': n_entries,
                     'playlist': playlist,

+                    'playlist_id': ie_result.get('id'),
+                    'playlist_title': ie_result.get('title'),

                     'playlist_index': i + playliststart,
                     'extractor': ie_result['extractor'],
                     'webpage_url': ie_result['webpage_url'],
                     'playlist_index': i + playliststart,
                     'extractor': ie_result['extractor'],
                     'webpage_url': ie_result['webpage_url'],


@@ -674,14 +696,20 @@ class YoutubeDL(object):
             ie_result['entries'] = playlist_results
             return ie_result
         elif result_type == 'compat_list':
             ie_result['entries'] = playlist_results
             return ie_result
         elif result_type == 'compat_list':

+            self.report_warning(
+                'Extractor %s returned a compat_list result. '
+                'It needs to be updated.' % ie_result.get('extractor'))
+

             def _fixup(r):
             def _fixup(r):

-                self.add_extra_info(r,
+                self.add_extra_info(
+                    r,

                     {
                         'extractor': ie_result['extractor'],
                         'webpage_url': ie_result['webpage_url'],
                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
                         'extractor_key': ie_result['extractor_key'],
                     {
                         'extractor': ie_result['extractor'],
                         'webpage_url': ie_result['webpage_url'],
                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
                         'extractor_key': ie_result['extractor_key'],

-                    })
+                    }
+                )

                 return r
             ie_result['entries'] = [
                 self.process_ie_result(_fixup(r), download, extra_info)
                 return r
             ie_result['entries'] = [
                 self.process_ie_result(_fixup(r), download, extra_info)


@@ -831,8 +859,15 @@ class YoutubeDL(object):
                         # Two formats have been requested like '137+139'
                         format_1, format_2 = rf.split('+')
                         formats_info = (self.select_format(format_1, formats),
                         # Two formats have been requested like '137+139'
                         format_1, format_2 = rf.split('+')
                         formats_info = (self.select_format(format_1, formats),

-                            self.select_format(format_2, formats))
+                                        self.select_format(format_2, formats))

                         if all(formats_info):
                         if all(formats_info):

+                            # The first format must contain the video and the
+                            # second the audio
+                            if formats_info[0].get('vcodec') == 'none':
+                                self.report_error('The first format must '
+                                                  'contain the video, try using '
+                                                  '"-f %s+%s"' % (format_2, format_1))
+                                return

                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,


@@ -977,7 +1012,7 @@ class YoutubeDL(object):
                     else:
                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
                     else:
                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:

-                                subfile.write(sub)
+                            subfile.write(sub)

                 except (OSError, IOError):
                     self.report_error('Cannot write subtitles file ' + sub_filename)
                     return
                 except (OSError, IOError):
                     self.report_error('Cannot write subtitles file ' + sub_filename)
                     return


@@ -989,7 +1024,7 @@ class YoutubeDL(object):
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                 try:
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                 try:

-                    write_json_file(info_dict, encodeFilename(infofn))
+                    write_json_file(info_dict, infofn)

                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return


@@ -1009,10 +1044,10 @@ class YoutubeDL(object):
                         with open(thumb_filename, 'wb') as thumbf:
                             shutil.copyfileobj(uf, thumbf)
                         self.to_screen('[%s] %s: Writing thumbnail to: %s' %
                         with open(thumb_filename, 'wb') as thumbf:
                             shutil.copyfileobj(uf, thumbf)
                         self.to_screen('[%s] %s: Writing thumbnail to: %s' %

-                            (info_dict['extractor'], info_dict['id'], thumb_filename))
+                                       (info_dict['extractor'], info_dict['id'], thumb_filename))

                     except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                         self.report_warning('Unable to download thumbnail "%s": %s' %
                     except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                         self.report_warning('Unable to download thumbnail "%s": %s' %

-                            (info_dict['thumbnail'], compat_str(err)))
+                                            (info_dict['thumbnail'], compat_str(err)))

 
         if not self.params.get('skip_download', False):
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 
         if not self.params.get('skip_download', False):
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):


@@ -1033,8 +1068,8 @@ class YoutubeDL(object):
                         if not merger._executable:
                             postprocessors = []
                             self.report_warning('You have requested multiple '
                         if not merger._executable:
                             postprocessors = []
                             self.report_warning('You have requested multiple '

-                                'formats but ffmpeg or avconv are not installed.'
-                                ' The formats won\'t be merged')
+                                                'formats but ffmpeg or avconv are not installed.'
+                                                ' The formats won\'t be merged')

                         else:
                             postprocessors = [merger]
                         for f in info_dict['requested_formats']:
                         else:
                             postprocessors = [merger]
                         for f in info_dict['requested_formats']:


@@ -1078,7 +1113,7 @@ class YoutubeDL(object):
 
         for url in url_list:
             try:
 
         for url in url_list:
             try:

-                #It also downloads the videos
+                # It also downloads the videos

                 res = self.extract_info(url)
             except UnavailableVideoError:
                 self.report_error('unable to download video')
                 res = self.extract_info(url)
             except UnavailableVideoError:
                 self.report_error('unable to download video')


@@ -1294,11 +1329,13 @@ class YoutubeDL(object):
             self.report_warning(
                 'Your Python is broken! Update to a newer and supported version')
 
             self.report_warning(
                 'Your Python is broken! Update to a newer and supported version')
 

+        stdout_encoding = getattr(
+            sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)

         encoding_str = (
             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
                 locale.getpreferredencoding(),
                 sys.getfilesystemencoding(),
         encoding_str = (
             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
                 locale.getpreferredencoding(),
                 sys.getfilesystemencoding(),

-                sys.stdout.encoding,
+                stdout_encoding,

                 self.get_encoding()))
         write_string(encoding_str, encoding=None)
 
                 self.get_encoding()))
         write_string(encoding_str, encoding=None)
 


@@ -1321,6 +1358,7 @@ class YoutubeDL(object):
             platform.python_version(), platform_name()))
 
         exe_versions = FFmpegPostProcessor.get_versions()
             platform.python_version(), platform_name()))
 
         exe_versions = FFmpegPostProcessor.get_versions()

+        exe_versions['rtmpdump'] = rtmpdump_version()

         exe_str = ', '.join(
             '%s %s' % (exe, v)
             for exe, v in sorted(exe_versions.items())
         exe_str = ', '.join(
             '%s %s' % (exe, v)
             for exe, v in sorted(exe_versions.items())