Merge tag 'upstream/2014.10.30'

[youtubedl] / youtube_dl / extractor / collegehumor.py

diff --git a/youtube_dl/extractor/collegehumor.py b/youtube_dl/extractor/collegehumor.py
index 8d4c93d6da91f4470c9809bf32dd0fbbe886c92b..6f866e7fcee7f401362b24d69db2285e22cfa6a4 100644 (file)
--- a/youtube_dl/extractor/collegehumor.py
+++ b/youtube_dl/extractor/collegehumor.py
@@ -1,87 +1,102 @@
+from __future__ import unicode_literals
+
+import json

 import re
 import re

-import xml.etree.ElementTree

 
 from .common import InfoExtractor
 
 from .common import InfoExtractor

-from ..utils import (
-    compat_urllib_parse_urlparse,
-    determine_ext,
-
-    ExtractorError,
-)
+from ..utils import int_or_none

 
 
 class CollegeHumorIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/(video|embed|e)/(?P<videoid>[0-9]+)/?(?P<shorttitle>.*)$'
 
     _TESTS = [{
 
 
 class CollegeHumorIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/(video|embed|e)/(?P<videoid>[0-9]+)/?(?P<shorttitle>.*)$'
 
     _TESTS = [{

-        u'url': u'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
-        u'file': u'6902724.mp4',
-        u'md5': u'1264c12ad95dca142a9f0bf7968105a0',
-        u'info_dict': {
-            u'title': u'Comic-Con Cosplay Catastrophe',
-            u'description': u'Fans get creative this year at San Diego.  Too creative.  And yes, that\'s really Joss Whedon.',
+        'url': 'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
+        'md5': 'dcc0f5c1c8be98dc33889a191f4c26bd',
+        'info_dict': {
+            'id': '6902724',
+            'ext': 'mp4',
+            'title': 'Comic-Con Cosplay Catastrophe',
+            'description': "Fans get creative this year at San Diego.  Too creative.  And yes, that's really Joss Whedon.",
+            'age_limit': 13,
+            'duration': 187,

         },
     },
     {
         },
     },
     {

-        u'url': u'http://www.collegehumor.com/video/3505939/font-conference',
-        u'file': u'3505939.mp4',
-        u'md5': u'c51ca16b82bb456a4397987791a835f5',
-        u'info_dict': {
-            u'title': u'Font Conference',
-            u'description': u'This video wasn\'t long enough, so we made it double-spaced.',
+        'url': 'http://www.collegehumor.com/video/3505939/font-conference',
+        'md5': '72fa701d8ef38664a4dbb9e2ab721816',
+        'info_dict': {
+            'id': '3505939',
+            'ext': 'mp4',
+            'title': 'Font Conference',
+            'description': "This video wasn't long enough, so we made it double-spaced.",
+            'age_limit': 10,
+            'duration': 179,

         },
         },

-    }]
+    },
+    # embedded youtube video
+    {
+        'url': 'http://www.collegehumor.com/embed/6950306',
+        'info_dict': {
+            'id': 'Z-bao9fg6Yc',
+            'ext': 'mp4',
+            'title': 'Young Americans Think President John F. Kennedy Died THIS MORNING IN A CAR ACCIDENT!!!',
+            'uploader': 'Mark Dice',
+            'uploader_id': 'MarkDice',
+            'description': 'md5:62c3dab9351fac7bb44b53b69511d87f',
+            'upload_date': '20140127',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': ['Youtube'],
+    },
+    ]

 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)

         video_id = mobj.group('videoid')
 
         video_id = mobj.group('videoid')
 

-        info = {
-            'id': video_id,
-            'uploader': None,
-            'upload_date': None,
-        }
+        jsonUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id + '.json'
+        data = json.loads(self._download_webpage(
+            jsonUrl, video_id, 'Downloading info JSON'))
+        vdata = data['video']
+        if vdata.get('youtubeId') is not None:
+            return {
+                '_type': 'url',
+                'url': vdata['youtubeId'],
+                'ie_key': 'Youtube',
+            }

 
 

-        self.report_extraction(video_id)
-        xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id
-        metaXml = self._download_webpage(xmlUrl, video_id,
-                                         u'Downloading info XML',
-                                         u'Unable to download video info XML')
+        AGE_LIMITS = {'nc17': 18, 'r': 18, 'pg13': 13, 'pg': 10, 'g': 0}
+        rating = vdata.get('rating')
+        if rating:
+            age_limit = AGE_LIMITS.get(rating.lower())
+        else:
+            age_limit = None  # None = No idea

 
 

-        mdoc = xml.etree.ElementTree.fromstring(metaXml)
-        try:
-            videoNode = mdoc.findall('./video')[0]
-            youtubeIdNode = videoNode.find('./youtubeID')
-            if youtubeIdNode is not None:
-                return self.url_result(youtubeIdNode.text, 'Youtube')
-            info['description'] = videoNode.findall('./description')[0].text
-            info['title'] = videoNode.findall('./caption')[0].text
-            info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
-            next_url = videoNode.findall('./file')[0].text
-        except IndexError:
-            raise ExtractorError(u'Invalid metadata XML file')
+        PREFS = {'high_quality': 2, 'low_quality': 0}
+        formats = []
+        for format_key in ('mp4', 'webm'):
+            for qname, qurl in vdata.get(format_key, {}).items():
+                formats.append({
+                    'format_id': format_key + '_' + qname,
+                    'url': qurl,
+                    'format': format_key,
+                    'preference': PREFS.get(qname),
+                })
+        self._sort_formats(formats)

 
 

-        if next_url.endswith(u'manifest.f4m'):
-            manifest_url = next_url + '?hdcore=2.10.3'
-            manifestXml = self._download_webpage(manifest_url, video_id,
-                                         u'Downloading XML manifest',
-                                         u'Unable to download video info XML')
+        duration = int_or_none(vdata.get('duration'), 1000)
+        like_count = int_or_none(vdata.get('likes'))

 
 

-            adoc = xml.etree.ElementTree.fromstring(manifestXml)
-            try:
-                media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0]
-                node_id = media_node.attrib['url']
-                video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
-            except IndexError as err:
-                raise ExtractorError(u'Invalid manifest file')
-            url_pr = compat_urllib_parse_urlparse(info['thumbnail'])
-            info['url'] = url_pr.scheme + '://' + url_pr.netloc + video_id[:-2].replace('.csmil','').replace(',','')
-            info['ext'] = 'mp4'
-        else:
-            # Old-style direct links
-            info['url'] = next_url
-            info['ext'] = determine_ext(info['url'])
-
-        return info
+        return {
+            'id': video_id,
+            'title': vdata['title'],
+            'description': vdata.get('description'),
+            'thumbnail': vdata.get('thumbnail'),
+            'formats': formats,
+            'age_limit': age_limit,
+            'duration': duration,
+            'like_count': like_count,
+        }