Merge tag 'upstream/2014.02.17'

[youtubedl] / youtube_dl / extractor / comedycentral.py

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 93d9e3d5e6ab96404034b83c10bcc4a28a87d78c..ed3986f313a149f0db4a69dc92762730297ced1a 100644 (file)
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -1,7 +1,9 @@
+from __future__ import unicode_literals
+

 import re
 import re

-import xml.etree.ElementTree

 
 from .common import InfoExtractor
 
 from .common import InfoExtractor

+from .mtv import MTVServicesInfoExtractor

 from ..utils import (
     compat_str,
     compat_urllib_parse,
 from ..utils import (
     compat_str,
     compat_urllib_parse,


@@ -11,8 +13,26 @@ from ..utils import (
 )
 
 
 )
 
 

-class ComedyCentralIE(InfoExtractor):
-    IE_DESC = u'The Daily Show / Colbert Report'
+class ComedyCentralIE(MTVServicesInfoExtractor):
+    _VALID_URL = r'''(?x)https?://(?:www\.)?comedycentral\.com/
+        (video-clips|episodes|cc-studios|video-collections)
+        /(?P<title>.*)'''
+    _FEED_URL = 'http://comedycentral.com/feeds/mrss/'
+
+    _TEST = {
+        'url': 'http://www.comedycentral.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother',
+        'md5': '4167875aae411f903b751a21f357f1ee',
+        'info_dict': {
+            'id': 'cef0cbb3-e776-4bc9-b62e-8016deccb354',
+            'ext': 'mp4',
+            'title': 'CC:Stand-Up|Greg Fitzsimmons: Life on Stage|Uncensored - Too Good of a Mother',
+            'description': 'After a certain point, breastfeeding becomes c**kblocking.',
+        },
+    }
+
+
+class ComedyCentralShowsIE(InfoExtractor):
+    IE_DESC = 'The Daily Show / Colbert Report'

     # urls can be abbreviations like :thedailyshow or :colbert
     # urls for episodes like:
     # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
     # urls can be abbreviations like :thedailyshow or :colbert
     # urls for episodes like:
     # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day


@@ -24,17 +44,19 @@ class ComedyCentralIE(InfoExtractor):
                          (full-episodes/(?P<episode>.*)|
                           (?P<clip>
                               (the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
                          (full-episodes/(?P<episode>.*)|
                           (?P<clip>
                               (the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))

-                              |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*)))))
+                              |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*)))|
+                          (?P<interview>
+                              extended-interviews/(?P<interID>[0-9]+)/playlist_tds_extended_(?P<interview_title>.*?)/.*?)))

                      $"""
     _TEST = {
                      $"""
     _TEST = {

-        u'url': u'http://www.thedailyshow.com/watch/thu-december-13-2012/kristen-stewart',
-        u'file': u'422212.mp4',
-        u'md5': u'4e2f5cb088a83cd8cdb7756132f9739d',
-        u'info_dict': {
-            u"upload_date": u"20121214", 
-            u"description": u"Kristen Stewart", 
-            u"uploader": u"thedailyshow", 
-            u"title": u"thedailyshow-kristen-stewart part 1"
+        'url': 'http://www.thedailyshow.com/watch/thu-december-13-2012/kristen-stewart',
+        'file': '422212.mp4',
+        'md5': '4e2f5cb088a83cd8cdb7756132f9739d',
+        'info_dict': {
+            "upload_date": "20121214",
+            "description": "Kristen Stewart",
+            "uploader": "thedailyshow",
+            "title": "thedailyshow-kristen-stewart part 1"

         }
     }
 
         }
     }
 


@@ -49,12 +71,12 @@ class ComedyCentralIE(InfoExtractor):
         '400': 'mp4',
     }
     _video_dimensions = {
         '400': 'mp4',
     }
     _video_dimensions = {

-        '3500': '1280x720',
-        '2200': '960x540',
-        '1700': '768x432',
-        '1200': '640x360',
-        '750': '512x288',
-        '400': '384x216',
+        '3500': (1280, 720),
+        '2200': (960, 540),
+        '1700': (768, 432),
+        '1200': (640, 360),
+        '750': (512, 288),
+        '400': (384, 216),

     }
 
     @classmethod
     }
 
     @classmethod


@@ -62,22 +84,24 @@ class ComedyCentralIE(InfoExtractor):
         """Receives a URL and returns True if suitable for this IE."""
         return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
 
         """Receives a URL and returns True if suitable for this IE."""
         return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
 

-    def _print_formats(self, formats):
-        print('Available formats:')
-        for x in formats:
-            print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
-
+    @staticmethod
+    def _transform_rtmp_url(rtmp_video_url):
+        m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\.comedystor/.*)$', rtmp_video_url)
+        if not m:
+            raise ExtractorError('Cannot transform RTMP url')
+        base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
+        return base + m.group('finalid')

 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url, re.VERBOSE)
         if mobj is None:
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url, re.VERBOSE)
         if mobj is None:

-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError('Invalid URL: %s' % url)

 
         if mobj.group('shortname'):
             if mobj.group('shortname') in ('tds', 'thedailyshow'):
 
         if mobj.group('shortname'):
             if mobj.group('shortname') in ('tds', 'thedailyshow'):

-                url = u'http://www.thedailyshow.com/full-episodes/'
+                url = 'http://www.thedailyshow.com/full-episodes/'

             else:
             else:

-                url = u'http://www.colbertnation.com/full-episodes/'
+                url = 'http://www.colbertnation.com/full-episodes/'

             mobj = re.match(self._VALID_URL, url, re.VERBOSE)
             assert mobj is not None
 
             mobj = re.match(self._VALID_URL, url, re.VERBOSE)
             assert mobj is not None
 


@@ -87,6 +111,9 @@ class ComedyCentralIE(InfoExtractor):
             else:
                 epTitle = mobj.group('cntitle')
             dlNewest = False
             else:
                 epTitle = mobj.group('cntitle')
             dlNewest = False

+        elif mobj.group('interview'):
+            epTitle = mobj.group('interview_title')
+            dlNewest = False

         else:
             dlNewest = not mobj.group('episode')
             if dlNewest:
         else:
             dlNewest = not mobj.group('episode')
             if dlNewest:


@@ -100,9 +127,9 @@ class ComedyCentralIE(InfoExtractor):
             url = htmlHandle.geturl()
             mobj = re.match(self._VALID_URL, url, re.VERBOSE)
             if mobj is None:
             url = htmlHandle.geturl()
             mobj = re.match(self._VALID_URL, url, re.VERBOSE)
             if mobj is None:

-                raise ExtractorError(u'Invalid redirected URL: ' + url)
+                raise ExtractorError('Invalid redirected URL: ' + url)

             if mobj.group('episode') == '':
             if mobj.group('episode') == '':

-                raise ExtractorError(u'Redirected URL is still not specific: ' + url)
+                raise ExtractorError('Redirected URL is still not specific: ' + url)

             epTitle = mobj.group('episode')
 
         mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', webpage)
             epTitle = mobj.group('episode')
 
         mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', webpage)


@@ -114,19 +141,18 @@ class ComedyCentralIE(InfoExtractor):
 
             altMovieParams = re.findall('data-mgid="([^"]*(?:episode|video).*?:.*?)"', webpage)
             if len(altMovieParams) == 0:
 
             altMovieParams = re.findall('data-mgid="([^"]*(?:episode|video).*?:.*?)"', webpage)
             if len(altMovieParams) == 0:

-                raise ExtractorError(u'unable to find Flash URL in webpage ' + url)
+                raise ExtractorError('unable to find Flash URL in webpage ' + url)

             else:
                 mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
 
         uri = mMovieParams[0][1]
         indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri})
             else:
                 mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
 
         uri = mMovieParams[0][1]
         indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri})

-        indexXml = self._download_webpage(indexUrl, epTitle,
-                                          u'Downloading show index',
-                                          u'unable to download episode index')
+        idoc = self._download_xml(indexUrl, epTitle,
+                                          'Downloading show index',
+                                          'unable to download episode index')

 
         results = []
 
 
         results = []
 

-        idoc = xml.etree.ElementTree.fromstring(indexXml)

         itemEls = idoc.findall('.//item')
         for partNum,itemEl in enumerate(itemEls):
             mediaId = itemEl.findall('./guid')[0].text
         itemEls = idoc.findall('.//item')
         for partNum,itemEl in enumerate(itemEls):
             mediaId = itemEl.findall('./guid')[0].text


@@ -137,53 +163,38 @@ class ComedyCentralIE(InfoExtractor):
 
             configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
                         compat_urllib_parse.urlencode({'uri': mediaId}))
 
             configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
                         compat_urllib_parse.urlencode({'uri': mediaId}))

-            configXml = self._download_webpage(configUrl, epTitle,
-                                               u'Downloading configuration for %s' % shortMediaId)
+            cdoc = self._download_xml(configUrl, epTitle,
+                                               'Downloading configuration for %s' % shortMediaId)

 
 

-            cdoc = xml.etree.ElementTree.fromstring(configXml)

             turls = []
             for rendition in cdoc.findall('.//rendition'):
                 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
                 turls.append(finfo)
 
             if len(turls) == 0:
             turls = []
             for rendition in cdoc.findall('.//rendition'):
                 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
                 turls.append(finfo)
 
             if len(turls) == 0:

-                self._downloader.report_error(u'unable to download ' + mediaId + ': No videos found')
+                self._downloader.report_error('unable to download ' + mediaId + ': No videos found')

                 continue
 
                 continue
 

-            if self._downloader.params.get('listformats', None):
-                self._print_formats([i[0] for i in turls])
-                return
-
-            # For now, just pick the highest bitrate
-            format,rtmp_video_url = turls[-1]
-
-            # Get the format arg from the arg stream
-            req_format = self._downloader.params.get('format', None)
-
-            # Select format if we can find one
-            for f,v in turls:
-                if f == req_format:
-                    format, rtmp_video_url = f, v
-                    break
-
-            m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp.comedystor/.*)$', rtmp_video_url)
-            if not m:
-                raise ExtractorError(u'Cannot transform RTMP url')
-            base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
-            video_url = base + m.group('finalid')
-
-            effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1)
-            info = {
+            formats = []
+            for format, rtmp_video_url in turls:
+                w, h = self._video_dimensions.get(format, (None, None))
+                formats.append({
+                    'url': self._transform_rtmp_url(rtmp_video_url),
+                    'ext': self._video_extensions.get(format, 'mp4'),
+                    'format_id': format,
+                    'height': h,
+                    'width': w,
+                })
+
+            effTitle = showId + '-' + epTitle + ' part ' + compat_str(partNum+1)
+            results.append({

                 'id': shortMediaId,
                 'id': shortMediaId,

-                'url': video_url,
+                'formats': formats,

                 'uploader': showId,
                 'upload_date': officialDate,
                 'title': effTitle,
                 'uploader': showId,
                 'upload_date': officialDate,
                 'title': effTitle,

-                'ext': 'mp4',
-                'format': format,

                 'thumbnail': None,
                 'description': compat_str(officialTitle),
                 'thumbnail': None,
                 'description': compat_str(officialTitle),

-            }
-            results.append(info)
+            })

 
         return results
 
         return results