Add bug number that we are closing.

[youtubedl] / youtube_dl / extractor / generic.py

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b633e896c6eb3d5b2158585d01ffba9615fe8f31..764070635160d96487baa53b4370b28e8a957316 100644 (file)
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -8,11 +8,13 @@ from ..utils import (
     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_request,
     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_request,

+    compat_urlparse,

 
     ExtractorError,
 )
 from .brightcove import BrightcoveIE
 
 
     ExtractorError,
 )
 from .brightcove import BrightcoveIE
 

+

 class GenericIE(InfoExtractor):
     IE_DESC = u'Generic downloader that works on some sites'
     _VALID_URL = r'.*'
 class GenericIE(InfoExtractor):
     IE_DESC = u'Generic downloader that works on some sites'
     _VALID_URL = r'.*'


@@ -23,21 +25,10 @@ class GenericIE(InfoExtractor):
             u'file': u'13601338388002.mp4',
             u'md5': u'85b90ccc9d73b4acd9138d3af4c27f89',
             u'info_dict': {
             u'file': u'13601338388002.mp4',
             u'md5': u'85b90ccc9d73b4acd9138d3af4c27f89',
             u'info_dict': {

-                u"uploader": u"www.hodiho.fr", 
+                u"uploader": u"www.hodiho.fr",

                 u"title": u"R\u00e9gis plante sa Jeep"
             }
         },
                 u"title": u"R\u00e9gis plante sa Jeep"
             }
         },

-        {
-            u'url': u'http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/',
-            u'file': u'2371591881001.mp4',
-            u'md5': u'9e80619e0a94663f0bdc849b4566af19',
-            u'note': u'Test Brightcove downloads and detection in GenericIE',
-            u'info_dict': {
-                u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
-                u'uploader': u'8TV',
-                u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
-            }
-        },

     ]
 
     def report_download_webpage(self, video_id):
     ]
 
     def report_download_webpage(self, video_id):


@@ -107,8 +98,18 @@ class GenericIE(InfoExtractor):
         return new_url
 
     def _real_extract(self, url):
         return new_url
 
     def _real_extract(self, url):

-        new_url = self._test_redirect(url)
-        if new_url: return [self.url_result(new_url)]
+        parsed_url = compat_urlparse.urlparse(url)
+        if not parsed_url.scheme:
+            self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
+            return self.url_result('http://' + url)
+
+        try:
+            new_url = self._test_redirect(url)
+            if new_url:
+                return [self.url_result(new_url)]
+        except compat_urllib_error.HTTPError:
+            # This may be a stupid server that doesn't like HEAD, our UA, or so
+            pass

 
         video_id = url.split('/')[-1]
         try:
 
         video_id = url.split('/')[-1]
         try:


@@ -119,7 +120,7 @@ class GenericIE(InfoExtractor):
             raise ExtractorError(u'Invalid URL: %s' % url)
 
         self.report_extraction(video_id)
             raise ExtractorError(u'Invalid URL: %s' % url)
 
         self.report_extraction(video_id)

-        # Look for BrigthCove:
+        # Look for BrightCove:

         m_brightcove = re.search(r'<object.+?class=([\'"]).*?BrightcoveExperience.*?\1.+?</object>', webpage, re.DOTALL)
         if m_brightcove is not None:
             self.to_screen(u'Brightcove video detected.')
         m_brightcove = re.search(r'<object.+?class=([\'"]).*?BrightcoveExperience.*?\1.+?</object>', webpage, re.DOTALL)
         if m_brightcove is not None:
             self.to_screen(u'Brightcove video detected.')


@@ -144,6 +145,9 @@ class GenericIE(InfoExtractor):
             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
             if m_video_type is not None:
                 mobj = re.search(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)
             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
             if m_video_type is not None:
                 mobj = re.search(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)

+        if mobj is None:
+            # HTML5 video
+            mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL)

         if mobj is None:
             raise ExtractorError(u'Invalid URL: %s' % url)
 
         if mobj is None:
             raise ExtractorError(u'Invalid URL: %s' % url)
 


@@ -152,8 +156,9 @@ class GenericIE(InfoExtractor):
         if mobj.group(1) is None:
             raise ExtractorError(u'Invalid URL: %s' % url)
 
         if mobj.group(1) is None:
             raise ExtractorError(u'Invalid URL: %s' % url)
 

-        video_url = compat_urllib_parse.unquote(mobj.group(1))
-        video_id = os.path.basename(video_url)
+        video_url = mobj.group(1)
+        video_url = compat_urlparse.urljoin(url, video_url)
+        video_id = compat_urllib_parse.unquote(os.path.basename(video_url))

 
         # here's a fun little line of code for you:
         video_extension = os.path.splitext(video_id)[1][1:]
 
         # here's a fun little line of code for you:
         video_extension = os.path.splitext(video_id)[1][1:]