Initiate release.

[youtubedl] / youtube_dl / extractor / dailymotion.py

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 71f5e03eea393b7733bf3bfeb4f2eeea5b21eb85..5d0bfe454c9bfe1a3e16d1273b18ed3be2f436b8 100644 (file)
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -8,11 +8,11 @@ from .subtitles import SubtitlesInfoExtractor
 from ..utils import (
     compat_urllib_request,
     compat_str,
 from ..utils import (
     compat_urllib_request,
     compat_str,

-    get_element_by_attribute,
-    get_element_by_id,

     orderedSet,
     orderedSet,

-
+    str_to_int,
+    int_or_none,

     ExtractorError,
     ExtractorError,

+    unescapeHTML,

 )
 
 class DailymotionBaseInfoExtractor(InfoExtractor):
 )
 
 class DailymotionBaseInfoExtractor(InfoExtractor):


@@ -27,7 +27,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
 class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
     """Information Extractor for Dailymotion"""
 
 class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
     """Information Extractor for Dailymotion"""
 

-    _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/(?:embed/)?video/([^/]+)'
+    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'

     IE_NAME = u'dailymotion'
 
     _FORMATS = [
     IE_NAME = u'dailymotion'
 
     _FORMATS = [


@@ -80,7 +80,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
         # Extract id and simplified title from URL
         mobj = re.match(self._VALID_URL, url)
 
         # Extract id and simplified title from URL
         mobj = re.match(self._VALID_URL, url)
 

-        video_id = mobj.group(1).split('_')[0].split('?')[0]
+        video_id = mobj.group('id')

 
         url = 'http://www.dailymotion.com/video/%s' % video_id
 
 
         url = 'http://www.dailymotion.com/video/%s' % video_id
 


@@ -100,10 +100,6 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
             self.to_screen(u'Vevo video detected: %s' % vevo_id)
             return self.url_result(u'vevo:%s' % vevo_id, ie='Vevo')
 
             self.to_screen(u'Vevo video detected: %s' % vevo_id)
             return self.url_result(u'vevo:%s' % vevo_id, ie='Vevo')
 

-        video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>',
-                                             # Looking for official user
-                                             r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'],
-                                            webpage, 'video uploader', fatal=False)

         age_limit = self._rta_search(webpage)
 
         video_upload_date = None
         age_limit = self._rta_search(webpage)
 
         video_upload_date = None


@@ -127,7 +123,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
             if video_url is not None:
                 m_size = re.search(r'H264-(\d+)x(\d+)', video_url)
                 if m_size is not None:
             if video_url is not None:
                 m_size = re.search(r'H264-(\d+)x(\d+)', video_url)
                 if m_size is not None:

-                    width, height = m_size.group(1), m_size.group(2)
+                    width, height = map(int_or_none, (m_size.group(1), m_size.group(2)))

                 else:
                     width, height = None, None
                 formats.append({
                 else:
                     width, height = None, None
                 formats.append({


@@ -146,15 +142,21 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
             self._list_available_subtitles(video_id, webpage)
             return
 
             self._list_available_subtitles(video_id, webpage)
             return
 

+        view_count = self._search_regex(
+            r'video_views_count[^>]+>\s+([\d\.,]+)', webpage, u'view count', fatal=False)
+        if view_count is not None:
+            view_count = str_to_int(view_count)
+

         return {
             'id':       video_id,
             'formats': formats,
         return {
             'id':       video_id,
             'formats': formats,

-            'uploader': video_uploader,
+            'uploader': info['owner.screenname'],

             'upload_date':  video_upload_date,
             'title':    self._og_search_title(webpage),
             'subtitles':    video_subtitles,
             'thumbnail': info['thumbnail_url'],
             'age_limit': age_limit,
             'upload_date':  video_upload_date,
             'title':    self._og_search_title(webpage),
             'subtitles':    video_subtitles,
             'thumbnail': info['thumbnail_url'],
             'age_limit': age_limit,

+            'view_count': view_count,

         }
 
     def _get_available_subtitles(self, video_id, webpage):
         }
 
     def _get_available_subtitles(self, video_id, webpage):


@@ -176,7 +178,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
 class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     IE_NAME = u'dailymotion:playlist'
     _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
 class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
     IE_NAME = u'dailymotion:playlist'
     _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'

-    _MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/playlist/.+?".*?>.*?</a>.*?</div>'
+    _MORE_PAGES_INDICATOR = r'(?s)<div class="pages[^"]*">.*?<a\s+class="[^"]*?icon-arrow_right[^"]*?"'

     _PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
 
     def _extract_entries(self, id):
     _PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
 
     def _extract_entries(self, id):


@@ -186,10 +188,9 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
             webpage = self._download_webpage(request,
                                              id, u'Downloading page %s' % pagenum)
 
             webpage = self._download_webpage(request,
                                              id, u'Downloading page %s' % pagenum)
 

-            playlist_el = get_element_by_attribute(u'class', u'row video_list', webpage)
-            video_ids.extend(re.findall(r'data-id="(.+?)"', playlist_el))
+            video_ids.extend(re.findall(r'data-xid="(.+?)"', webpage))

 
 

-            if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
+            if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:

                 break
         return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
                    for video_id in orderedSet(video_ids)]
                 break
         return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
                    for video_id in orderedSet(video_ids)]


@@ -199,26 +200,26 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
         playlist_id = mobj.group('id')
         webpage = self._download_webpage(url, playlist_id)
 
         playlist_id = mobj.group('id')
         webpage = self._download_webpage(url, playlist_id)
 

-        return {'_type': 'playlist',
-                'id': playlist_id,
-                'title': get_element_by_id(u'playlist_name', webpage),
-                'entries': self._extract_entries(playlist_id),
-                }
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'title': self._og_search_title(webpage),
+            'entries': self._extract_entries(playlist_id),
+        }

 
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = u'dailymotion:user'
 
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = u'dailymotion:user'

-    _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/user/(?P<user>[^/]+)'
-    _MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/user/.+?".*?>.*?</a>.*?</div>'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/user/(?P<user>[^/]+)'

     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         user = mobj.group('user')
         webpage = self._download_webpage(url, user)
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         user = mobj.group('user')
         webpage = self._download_webpage(url, user)

-        full_user = self._html_search_regex(
-            r'<a class="label" href="/%s".*?>(.*?)</' % re.escape(user),
-            webpage, u'user', flags=re.DOTALL)
+        full_user = unescapeHTML(self._html_search_regex(
+            r'<a class="nav-image" title="([^"]+)" href="/%s">' % re.escape(user),
+            webpage, u'user', flags=re.DOTALL))

 
         return {
             '_type': 'playlist',
 
         return {
             '_type': 'playlist',