New upstream version 2017.11.06

[youtubedl] / youtube_dl / extractor / condenast.py

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 8d8f605980bdf531c49a0c5d5067223d1f41dc4d..ed278fefc67089f44760ed8e0635cf1c5cbb1fde 100644 (file)
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -9,12 +9,12 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
     compat_urlparse,
 )
 from ..utils import (

-    orderedSet,
-    remove_end,
-    extract_attributes,
-    mimetype2ext,

     determine_ext,
     determine_ext,

+    extract_attributes,

     int_or_none,
     int_or_none,

+    js_to_json,
+    mimetype2ext,
+    orderedSet,

     parse_iso8601,
 )
 
     parse_iso8601,
 )
 


@@ -49,10 +49,17 @@ class CondeNastIE(InfoExtractor):
         'wmagazine': 'W Magazine',
     }
 
         'wmagazine': 'W Magazine',
     }
 

-    _VALID_URL = r'https?://(?:video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed(?:js)?)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
+    _VALID_URL = r'''(?x)https?://(?:video|www|player(?:-backend)?)\.(?:%s)\.com/
+        (?:
+            (?:
+                embed(?:js)?|
+                (?:script|inline)/video
+            )/(?P<id>[0-9a-f]{24})(?:/(?P<player_id>[0-9a-f]{24}))?(?:.+?\btarget=(?P<target>[^&]+))?|
+            (?P<type>watch|series|video)/(?P<display_id>[^/?#]+)
+        )''' % '|'.join(_SITES.keys())

     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 

-    EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed(?:js)?)/.+?' % '|'.join(_SITES.keys())
+    EMBED_URL = r'(?:https?:)?//player(?:-backend)?\.(?:%s)\.com/(?:embed(?:js)?|(?:script|inline)/video)/.+?' % '|'.join(_SITES.keys())

 
     _TESTS = [{
         'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
 
     _TESTS = [{
         'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',


@@ -66,6 +73,16 @@ class CondeNastIE(InfoExtractor):
             'upload_date': '20130314',
             'timestamp': 1363219200,
         }
             'upload_date': '20130314',
             'timestamp': 1363219200,
         }

+    }, {
+        'url': 'http://video.gq.com/watch/the-closer-with-keith-olbermann-the-only-true-surprise-trump-s-an-idiot?c=series',
+        'info_dict': {
+            'id': '58d1865bfd2e6126e2000015',
+            'ext': 'mp4',
+            'title': 'The Only True Surprise? Trump’s an Idiot',
+            'uploader': 'gq',
+            'upload_date': '20170321',
+            'timestamp': 1490126427,
+        },

     }, {
         # JS embed
         'url': 'http://player.cnevids.com/embedjs/55f9cf8b61646d1acf00000c/5511d76261646d5566020000.js',
     }, {
         # JS embed
         'url': 'http://player.cnevids.com/embedjs/55f9cf8b61646d1acf00000c/5511d76261646d5566020000.js',


@@ -78,6 +95,12 @@ class CondeNastIE(InfoExtractor):
             'upload_date': '20150916',
             'timestamp': 1442434955,
         }
             'upload_date': '20150916',
             'timestamp': 1442434955,
         }

+    }, {
+        'url': 'https://player.cnevids.com/inline/video/59138decb57ac36b83000005.js?target=js-cne-player',
+        'only_matching': True,
+    }, {
+        'url': 'http://player-backend.cnevids.com/script/video/59138decb57ac36b83000005.js',
+        'only_matching': True,

     }]
 
     def _extract_series(self, url, webpage):
     }]
 
     def _extract_series(self, url, webpage):


@@ -93,16 +116,16 @@ class CondeNastIE(InfoExtractor):
         entries = [self.url_result(build_url(path), 'CondeNast') for path in paths]
         return self.playlist_result(entries, playlist_title=title)
 
         entries = [self.url_result(build_url(path), 'CondeNast') for path in paths]
         return self.playlist_result(entries, playlist_title=title)
 

-    def _extract_video(self, webpage, url_type):
-        query = {}
-        params = self._search_regex(
-            r'(?s)var params = {(.+?)}[;,]', webpage, 'player params', default=None)
-        if params:
-            query.update({
-                'videoId': self._search_regex(r'videoId: [\'"](.+?)[\'"]', params, 'video id'),
-                'playerId': self._search_regex(r'playerId: [\'"](.+?)[\'"]', params, 'player id'),
-                'target': self._search_regex(r'target: [\'"](.+?)[\'"]', params, 'target'),
-            })
+    def _extract_video_params(self, webpage, display_id):
+        query = self._parse_json(
+            self._search_regex(
+                r'(?s)var\s+params\s*=\s*({.+?})[;,]', webpage, 'player params',
+                default='{}'),
+            display_id, transform_source=js_to_json, fatal=False)
+        if query:
+            query['videoId'] = self._search_regex(
+                r'(?:data-video-id=|currentVideoId\s*=\s*)["\']([\da-f]+)',
+                webpage, 'video id', default=None)

         else:
             params = extract_attributes(self._search_regex(
                 r'(<[^>]+data-js="video-player"[^>]+>)',
         else:
             params = extract_attributes(self._search_regex(
                 r'(<[^>]+data-js="video-player"[^>]+>)',


@@ -112,28 +135,58 @@ class CondeNastIE(InfoExtractor):
                 'playerId': params['data-player'],
                 'target': params['id'],
             })
                 'playerId': params['data-player'],
                 'target': params['id'],
             })

-        video_id = query['videoId']
+        return query
+
+    def _extract_video(self, params):
+        video_id = params['videoId']
+

         video_info = None
         video_info = None

-        info_page = self._download_webpage(
-            'http://player.cnevids.com/player/video.js',
-            video_id, 'Downloading video info', query=query, fatal=False)
+
+        # New API path
+        query = params.copy()
+        query['embedType'] = 'inline'
+        info_page = self._download_json(
+            'http://player.cnevids.com/embed-api.json', video_id,
+            'Downloading embed info', fatal=False, query=query)
+
+        # Old fallbacks
+        if not info_page:
+            if params.get('playerId'):
+                info_page = self._download_json(
+                    'http://player.cnevids.com/player/video.js', video_id,
+                    'Downloading video info', fatal=False, query=params)

         if info_page:
         if info_page:

-            video_info = self._parse_json(self._search_regex(
-                r'loadCallback\(({.+})\)', info_page, 'video info'), video_id)['video']
-        else:
+            video_info = info_page.get('video')
+        if not video_info:

             info_page = self._download_webpage(
                 'http://player.cnevids.com/player/loader.js',
             info_page = self._download_webpage(
                 'http://player.cnevids.com/player/loader.js',

-                video_id, 'Downloading loader info', query=query)
-            video_info = self._parse_json(self._search_regex(
-                r'var\s+video\s*=\s*({.+?});', info_page, 'video info'), video_id)
+                video_id, 'Downloading loader info', query=params)
+        if not video_info:
+            info_page = self._download_webpage(
+                'https://player.cnevids.com/inline/video/%s.js' % video_id,
+                video_id, 'Downloading inline info', query={
+                    'target': params.get('target', 'embedplayer')
+                })
+
+        if not video_info:
+            video_info = self._parse_json(
+                self._search_regex(
+                    r'(?s)var\s+config\s*=\s*({.+?});', info_page, 'config'),
+                video_id, transform_source=js_to_json)['video']
+

         title = video_info['title']
 
         formats = []
         title = video_info['title']
 
         formats = []

-        for fdata in video_info.get('sources', [{}])[0]:
+        for fdata in video_info['sources']:

             src = fdata.get('src')
             if not src:
                 continue
             ext = mimetype2ext(fdata.get('type')) or determine_ext(src)
             src = fdata.get('src')
             if not src:
                 continue
             ext = mimetype2ext(fdata.get('type')) or determine_ext(src)

+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+                continue

             quality = fdata.get('quality')
             formats.append({
                 'format_id': ext + ('-%s' % quality if quality else ''),
             quality = fdata.get('quality')
             formats.append({
                 'format_id': ext + ('-%s' % quality if quality else ''),


@@ -143,9 +196,7 @@ class CondeNastIE(InfoExtractor):
             })
         self._sort_formats(formats)
 
             })
         self._sort_formats(formats)
 

-        info = self._search_json_ld(
-            webpage, video_id, fatal=False) if url_type != 'embed' else {}
-        info.update({
+        return {

             'id': video_id,
             'formats': formats,
             'title': title,
             'id': video_id,
             'formats': formats,
             'title': title,


@@ -156,23 +207,26 @@ class CondeNastIE(InfoExtractor):
             'series': video_info.get('series_title'),
             'season': video_info.get('season_title'),
             'timestamp': parse_iso8601(video_info.get('premiere_date')),
             'series': video_info.get('series_title'),
             'season': video_info.get('season_title'),
             'timestamp': parse_iso8601(video_info.get('premiere_date')),

-        })
-        return info
+            'categories': video_info.get('categories'),
+        }

 
     def _real_extract(self, url):
 
     def _real_extract(self, url):

-        site, url_type, item_id = re.match(self._VALID_URL, url).groups()
+        video_id, player_id, target, url_type, display_id = re.match(self._VALID_URL, url).groups()

 
 

-        # Convert JS embed to regular embed
-        if url_type == 'embedjs':
-            parsed_url = compat_urlparse.urlparse(url)
-            url = compat_urlparse.urlunparse(parsed_url._replace(
-                path=remove_end(parsed_url.path, '.js').replace('/embedjs/', '/embed/')))
-            url_type = 'embed'
+        if video_id:
+            return self._extract_video({
+                'videoId': video_id,
+                'playerId': player_id,
+                'target': target,
+            })

 
 

-        self.to_screen('Extracting from %s with the Condé Nast extractor' % self._SITES[site])
-        webpage = self._download_webpage(url, item_id)
+        webpage = self._download_webpage(url, display_id)

 
         if url_type == 'series':
             return self._extract_series(url, webpage)
         else:
 
         if url_type == 'series':
             return self._extract_series(url, webpage)
         else:

-            return self._extract_video(webpage, url_type)
+            params = self._extract_video_params(webpage, display_id)
+            info = self._search_json_ld(
+                webpage, display_id, fatal=False)
+            info.update(self._extract_video(params))
+            return info