Merge tag 'upstream/2016.12.01'

[youtubedl] / youtube_dl / extractor / moniker.py

diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 79bb2ca5935cb7ad888081540696e256c31faa98..b208820fe64970b3f7b362dba13c534a6955686d 100644 (file)
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -6,14 +6,16 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
 
 from .common import InfoExtractor
 from ..utils import (

-    compat_urllib_parse,
-    compat_urllib_request,
+    ExtractorError,
+    remove_start,
+    sanitized_Request,
+    urlencode_postdata,

 )
 
 
 class MonikerIE(InfoExtractor):
     IE_DESC = 'allmyvideos.net and vidspot.net'
 )
 
 
 class MonikerIE(InfoExtractor):
     IE_DESC = 'allmyvideos.net and vidspot.net'

-    _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?P<id>[a-zA-Z0-9_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?:(?:2|v)/v-)?(?P<id>[a-zA-Z0-9_-]+)'

 
     _TESTS = [{
         'url': 'http://allmyvideos.net/jih3nce3x6wn',
 
     _TESTS = [{
         'url': 'http://allmyvideos.net/jih3nce3x6wn',


@@ -23,6 +25,14 @@ class MonikerIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'youtube-dl test video',
         },
             'ext': 'mp4',
             'title': 'youtube-dl test video',
         },

+    }, {
+        'url': 'http://allmyvideos.net/embed-jih3nce3x6wn',
+        'md5': '710883dee1bfc370ecf9fa6a89307c88',
+        'info_dict': {
+            'id': 'jih3nce3x6wn',
+            'ext': 'mp4',
+            'title': 'youtube-dl test video',
+        },

     }, {
         'url': 'http://vidspot.net/l2ngsmhs8ci5',
         'md5': '710883dee1bfc370ecf9fa6a89307c88',
     }, {
         'url': 'http://vidspot.net/l2ngsmhs8ci5',
         'md5': '710883dee1bfc370ecf9fa6a89307c88',


@@ -34,27 +44,62 @@ class MonikerIE(InfoExtractor):
     }, {
         'url': 'https://www.vidspot.net/l2ngsmhs8ci5',
         'only_matching': True,
     }, {
         'url': 'https://www.vidspot.net/l2ngsmhs8ci5',
         'only_matching': True,

+    }, {
+        'url': 'http://vidspot.net/2/v-ywDf99',
+        'md5': '5f8254ce12df30479428b0152fb8e7ba',
+        'info_dict': {
+            'id': 'ywDf99',
+            'ext': 'mp4',
+            'title': 'IL FAIT LE MALIN EN PORSHE CAYENNE ( mais pas pour longtemps)',
+            'description': 'IL FAIT LE MALIN EN PORSHE CAYENNE.',
+        },
+    }, {
+        'url': 'http://allmyvideos.net/v/v-HXZm5t',
+        'only_matching': True,

     }]
 
     def _real_extract(self, url):
     }]
 
     def _real_extract(self, url):

-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        orig_video_id = self._match_id(url)
+        video_id = remove_start(orig_video_id, 'embed-')
+        url = url.replace(orig_video_id, video_id)
+        assert re.match(self._VALID_URL, url) is not None

         orig_webpage = self._download_webpage(url, video_id)
         orig_webpage = self._download_webpage(url, video_id)

-        fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
-        data = dict(fields)

 
 

-        post = compat_urllib_parse.urlencode(data)
-        headers = {
-            b'Content-Type': b'application/x-www-form-urlencoded',
-        }
-        req = compat_urllib_request.Request(url, post, headers)
-        webpage = self._download_webpage(
-            req, video_id, note='Downloading video page ...')
+        if '>File Not Found<' in orig_webpage:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
+        error = self._search_regex(
+            r'class="err">([^<]+)<', orig_webpage, 'error', default=None)
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
+
+        builtin_url = self._search_regex(
+            r'<iframe[^>]+src=(["\'])(?P<url>.+?/builtin-.+?)\1',
+            orig_webpage, 'builtin URL', default=None, group='url')
+
+        if builtin_url:
+            req = sanitized_Request(builtin_url)
+            req.add_header('Referer', url)
+            webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
+            title = self._og_search_title(orig_webpage).strip()
+            description = self._og_search_description(orig_webpage).strip()
+        else:
+            fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
+            data = dict(fields)
+
+            post = urlencode_postdata(data)
+            headers = {
+                b'Content-Type': b'application/x-www-form-urlencoded',
+            }
+            req = sanitized_Request(url, post, headers)
+            webpage = self._download_webpage(
+                req, video_id, note='Downloading video page ...')

 
 

-        title = os.path.splitext(data['fname'])[0]
+            title = os.path.splitext(data['fname'])[0]
+            description = None

 
 

-        #Could be several links with different quality
+        # Could be several links with different quality

         links = re.findall(r'"file" : "?(.+?)",', webpage)
         # Assume the links are ordered in quality
         formats = [{
         links = re.findall(r'"file" : "?(.+?)",', webpage)
         # Assume the links are ordered in quality
         formats = [{


@@ -66,5 +111,6 @@ class MonikerIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
         return {
             'id': video_id,
             'title': title,

+            'description': description,

             'formats': formats,
         }
             'formats': formats,
         }