Update changelog.

[youtubedl] / youtube_dl / extractor / dailymotion.py

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index bc782393173f315b096a640bf20bfa45cb89eb8d..2e6226ea0774af2e636cbc4b4a4ca9f1ecb763a3 100644 (file)
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -7,15 +7,13 @@ import itertools
 
 from .common import InfoExtractor
 
 
 from .common import InfoExtractor
 

-from ..compat import (
-    compat_str,
-    compat_urllib_request,
-)

 from ..utils import (
 from ..utils import (

-    ExtractorError,

     determine_ext,
     determine_ext,

+    error_to_compat_str,
+    ExtractorError,

     int_or_none,
     parse_iso8601,
     int_or_none,
     parse_iso8601,

+    sanitized_Request,

     str_to_int,
     unescapeHTML,
 )
     str_to_int,
     unescapeHTML,
 )


@@ -25,7 +23,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
     @staticmethod
     def _build_request(url):
         """Build a request with the family filter disabled"""
     @staticmethod
     def _build_request(url):
         """Build a request with the family filter disabled"""

-        request = compat_urllib_request.Request(url)
+        request = sanitized_Request(url)

         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 
         request.add_header('Cookie', 'family_filter=off; ff=off')
         return request
 


@@ -39,7 +37,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
 
 
 class DailymotionIE(DailymotionBaseInfoExtractor):
 
 
 class DailymotionIE(DailymotionBaseInfoExtractor):

-    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
+    _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(?:embed|swf|#)/)?video/(?P<id>[^/?_]+)'

     IE_NAME = 'dailymotion'
 
     _FORMATS = [
     IE_NAME = 'dailymotion'
 
     _FORMATS = [


@@ -101,6 +99,15 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         {
             'url': 'http://www.dailymotion.com/video/xhza0o',
             'only_matching': True,
         {
             'url': 'http://www.dailymotion.com/video/xhza0o',
             'only_matching': True,

+        },
+        # with subtitles
+        {
+            'url': 'http://www.dailymotion.com/video/x20su5f_the-power-of-nightmares-1-the-rise-of-the-politics-of-fear-bbc-2004_news',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.dailymotion.com/swf/video/x3n92nf',
+            'only_matching': True,

         }
     ]
 
         }
     ]
 


@@ -115,16 +122,21 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
         description = self._og_search_description(webpage) or self._html_search_meta(
             'description', webpage, 'description')
 
         description = self._og_search_description(webpage) or self._html_search_meta(
             'description', webpage, 'description')
 

-        view_count = str_to_int(self._search_regex(
-            [r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:(\d+)"',
-             r'video_views_count[^>]+>\s+([\d\.,]+)'],
-            webpage, 'view count', fatal=False))
+        view_count_str = self._search_regex(
+            (r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserPlays:([\s\d,.]+)"',
+             r'video_views_count[^>]+>\s+([\s\d\,.]+)'),
+            webpage, 'view count', fatal=False)
+        if view_count_str:
+            view_count_str = re.sub(r'\s', '', view_count_str)
+        view_count = str_to_int(view_count_str)

         comment_count = int_or_none(self._search_regex(
             r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserComments:(\d+)"',
             webpage, 'comment count', fatal=False))
 
         player_v5 = self._search_regex(
         comment_count = int_or_none(self._search_regex(
             r'<meta[^>]+itemprop="interactionCount"[^>]+content="UserComments:(\d+)"',
             webpage, 'comment count', fatal=False))
 
         player_v5 = self._search_regex(

-            [r'buildPlayer\(({.+?})\);', r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);'],
+            [r'buildPlayer\(({.+?})\);\n',  # See https://github.com/rg3/youtube-dl/issues/7826
+             r'playerV5\s*=\s*dmp\.create\([^,]+?,\s*({.+?})\);',
+             r'buildPlayer\(({.+?})\);'],

             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)
             webpage, 'player v5', default=None)
         if player_v5:
             player = self._parse_json(player_v5, video_id)


@@ -143,19 +155,16 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                         continue
                     ext = determine_ext(media_url)
                     if type_ == 'application/x-mpegURL' or ext == 'm3u8':
                         continue
                     ext = determine_ext(media_url)
                     if type_ == 'application/x-mpegURL' or ext == 'm3u8':

-                        m3u8_formats = self._extract_m3u8_formats(
-                            media_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
-                        if m3u8_formats:
-                            formats.extend(m3u8_formats)
+                        formats.extend(self._extract_m3u8_formats(
+                            media_url, video_id, 'mp4', preference=-1,
+                            m3u8_id='hls', fatal=False))

                     elif type_ == 'application/f4m' or ext == 'f4m':
                     elif type_ == 'application/f4m' or ext == 'f4m':

-                        f4m_formats = self._extract_f4m_formats(
-                            media_url, video_id, preference=-1, f4m_id='hds', fatal=False)
-                        if f4m_formats:
-                            formats.extend(f4m_formats)
+                        formats.extend(self._extract_f4m_formats(
+                            media_url, video_id, preference=-1, f4m_id='hds', fatal=False))

                     else:
                         f = {
                             'url': media_url,
                     else:
                         f = {
                             'url': media_url,

-                            'format_id': quality,
+                            'format_id': 'http-%s' % quality,

                         }
                         m = re.search(r'H264-(?P<width>\d+)x(?P<height>\d+)', media_url)
                         if m:
                         }
                         m = re.search(r'H264-(?P<width>\d+)x(?P<height>\d+)', media_url)
                         if m:


@@ -174,11 +183,13 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
             uploader_id = metadata.get('owner', {}).get('id')
 
             subtitles = {}
             uploader_id = metadata.get('owner', {}).get('id')
 
             subtitles = {}

-            for subtitle_lang, subtitle in metadata.get('subtitles', {}).get('data', {}).items():
-                subtitles[subtitle_lang] = [{
-                    'ext': determine_ext(subtitle_url),
-                    'url': subtitle_url,
-                } for subtitle_url in subtitle.get('urls', [])]
+            subtitles_data = metadata.get('subtitles', {}).get('data', {})
+            if subtitles_data and isinstance(subtitles_data, dict):
+                for subtitle_lang, subtitle in subtitles_data.items():
+                    subtitles[subtitle_lang] = [{
+                        'ext': determine_ext(subtitle_url),
+                        'url': subtitle_url,
+                    } for subtitle_url in subtitle.get('urls', [])]

 
             return {
                 'id': video_id,
 
             return {
                 'id': video_id,


@@ -271,7 +282,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
                 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
                 video_id, note=False)
         except ExtractorError as err:

-            self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))

             return {}
         info = json.loads(sub_list)
         if (info['total'] > 0):
             return {}
         info = json.loads(sub_list)
         if (info['total'] > 0):


@@ -332,7 +343,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'
 
 class DailymotionUserIE(DailymotionPlaylistIE):
     IE_NAME = 'dailymotion:user'

-    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dailymotion\.[a-z]{2,3}/(?!(?:embed|swf|#|video|playlist)/)(?:(?:old/)?user/)?(?P<user>[^/]+)'

     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',
     _PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
     _TESTS = [{
         'url': 'https://www.dailymotion.com/user/nqtv',


@@ -388,13 +399,13 @@ class DailymotionCloudIE(DailymotionBaseInfoExtractor):
     }]
 
     @classmethod
     }]
 
     @classmethod

-    def _extract_dmcloud_url(self, webpage):
-        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % self._VALID_EMBED_URL, webpage)
+    def _extract_dmcloud_url(cls, webpage):
+        mobj = re.search(r'<iframe[^>]+src=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL, webpage)

         if mobj:
             return mobj.group(1)
 
         mobj = re.search(
         if mobj:
             return mobj.group(1)
 
         mobj = re.search(

-            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % self._VALID_EMBED_URL,
+            r'<input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=[\'"](%s)[\'"]' % cls._VALID_EMBED_URL,

             webpage)
         if mobj:
             return mobj.group(1)
             webpage)
         if mobj:
             return mobj.group(1)