debian/changelog: Update with bugs being closed.

[youtubedl] / youtube_dl / extractor / generic.py

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dc4dea4adf63937722a1bf81ead5e10fe09f34e3..69e0a7bd271dd0965f5c1f6f9c3a7cdce7a3da0b 100644 (file)
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -11,6 +11,8 @@ from ..utils import (
     compat_urlparse,
 
     ExtractorError,
     compat_urlparse,
 
     ExtractorError,

+    smuggle_url,
+    unescapeHTML,

 )
 from .brightcove import BrightcoveIE
 
 )
 from .brightcove import BrightcoveIE
 


@@ -29,17 +31,17 @@ class GenericIE(InfoExtractor):
                 u"title": u"R\u00e9gis plante sa Jeep"
             }
         },
                 u"title": u"R\u00e9gis plante sa Jeep"
             }
         },

+        # embedded vimeo video

         {
         {

-            u'url': u'http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/',
-            u'file': u'2371591881001.mp4',
-            u'md5': u'9e80619e0a94663f0bdc849b4566af19',
-            u'note': u'Test Brightcove downloads and detection in GenericIE',
+            u'url': u'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
+            u'file': u'22444065.mp4',
+            u'md5': u'2903896e23df39722c33f015af0666e2',

             u'info_dict': {
             u'info_dict': {

-                u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
-                u'uploader': u'8TV',
-                u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
+                u'title': u'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011',
+                u"uploader_id": u"skillsmatter",
+                u"uploader": u"Skills Matter",

             }
             }

-        },
+        }

     ]
 
     def report_download_webpage(self, video_id):
     ]
 
     def report_download_webpage(self, video_id):


@@ -109,6 +111,11 @@ class GenericIE(InfoExtractor):
         return new_url
 
     def _real_extract(self, url):
         return new_url
 
     def _real_extract(self, url):

+        parsed_url = compat_urlparse.urlparse(url)
+        if not parsed_url.scheme:
+            self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
+            return self.url_result('http://' + url)
+

         try:
             new_url = self._test_redirect(url)
             if new_url:
         try:
             new_url = self._test_redirect(url)
             if new_url:


@@ -123,16 +130,31 @@ class GenericIE(InfoExtractor):
         except ValueError:
             # since this is the last-resort InfoExtractor, if
             # this error is thrown, it'll be thrown here
         except ValueError:
             # since this is the last-resort InfoExtractor, if
             # this error is thrown, it'll be thrown here

-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError(u'Failed to download URL: %s' % url)

 
         self.report_extraction(video_id)
         # Look for BrightCove:
 
         self.report_extraction(video_id)
         # Look for BrightCove:

-        m_brightcove = re.search(r'<object.+?class=([\'"]).*?BrightcoveExperience.*?\1.+?</object>', webpage, re.DOTALL)
+        m_brightcove = re.search(r'<object[^>]+?class=([\'"])[^>]*?BrightcoveExperience.*?\1.+?</object>', webpage, re.DOTALL)

         if m_brightcove is not None:
             self.to_screen(u'Brightcove video detected.')
             bc_url = BrightcoveIE._build_brighcove_url(m_brightcove.group())
             return self.url_result(bc_url, 'Brightcove')
 
         if m_brightcove is not None:
             self.to_screen(u'Brightcove video detected.')
             bc_url = BrightcoveIE._build_brighcove_url(m_brightcove.group())
             return self.url_result(bc_url, 'Brightcove')
 

+        # Look for embedded Vimeo player
+        mobj = re.search(
+            r'<iframe[^>]+?src="(https?://player.vimeo.com/video/.+?)"', webpage)
+        if mobj:
+            player_url = unescapeHTML(mobj.group(1))
+            surl = smuggle_url(player_url, {'Referer': url})
+            return self.url_result(surl, 'Vimeo')
+
+        # Look for embedded YouTube player
+        mobj = re.search(
+            r'<iframe[^>]+?src="(https?://(?:www\.)?youtube.com/embed/.+?)"', webpage)
+        if mobj:
+            surl = unescapeHTML(mobj.group(1))
+            return self.url_result(surl, 'Youtube')
+

         # Start with something easy: JW Player in SWFObject
         mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
         if mobj is None:
         # Start with something easy: JW Player in SWFObject
         mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
         if mobj is None:


@@ -153,18 +175,18 @@ class GenericIE(InfoExtractor):
                 mobj = re.search(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)
         if mobj is None:
             # HTML5 video
                 mobj = re.search(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)
         if mobj is None:
             # HTML5 video

-            mobj = re.search(r'<video[^<]*>.*?<source .*?src="([^"]+)"', webpage, flags=re.DOTALL)
+            mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL)

         if mobj is None:
         if mobj is None:

-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError(u'Unsupported URL: %s' % url)

 
         # It's possible that one of the regexes
         # matched, but returned an empty group:
         if mobj.group(1) is None:
 
         # It's possible that one of the regexes
         # matched, but returned an empty group:
         if mobj.group(1) is None:

-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError(u'Did not find a valid video URL at %s' % url)

 
 

-        video_url = compat_urllib_parse.unquote(mobj.group(1))
+        video_url = mobj.group(1)

         video_url = compat_urlparse.urljoin(url, video_url)
         video_url = compat_urlparse.urljoin(url, video_url)

-        video_id = os.path.basename(video_url)
+        video_id = compat_urllib_parse.unquote(os.path.basename(video_url))

 
         # here's a fun little line of code for you:
         video_extension = os.path.splitext(video_id)[1][1:]
 
         # here's a fun little line of code for you:
         video_extension = os.path.splitext(video_id)[1][1:]