Update upstream source from tag 'upstream/2018.03.14'

[youtubedl] / youtube_dl / extractor / sendtonews.py

diff --git a/youtube_dl/extractor/sendtonews.py b/youtube_dl/extractor/sendtonews.py
index 1c636f67237af84d29b36d8340579811b692ef4f..9d9652949bb64ca2a15c02b3e0733dd9b7f42493 100644 (file)
--- a/youtube_dl/extractor/sendtonews.py
+++ b/youtube_dl/extractor/sendtonews.py
@@ -3,34 +3,47 @@ from __future__ import unicode_literals
 
 import re
 
 
 import re
 

-from .jwplatform import JWPlatformBaseIE
-from ..compat import compat_parse_qs
+from .common import InfoExtractor

 from ..utils import (
 from ..utils import (

-    ExtractorError,
-    parse_duration,
+    float_or_none,
+    parse_iso8601,
+    update_url_query,
+    int_or_none,
+    determine_protocol,
+    unescapeHTML,

 )
 
 
 )
 
 

-class SendtoNewsIE(JWPlatformBaseIE):
-    _VALID_URL = r'https?://embed\.sendtonews\.com/player/embed\.php\?(?P<query>[^#]+)'
+class SendtoNewsIE(InfoExtractor):
+    _VALID_URL = r'https?://embed\.sendtonews\.com/player2/embedplayer\.php\?.*\bSC=(?P<id>[0-9A-Za-z-]+)'

 
     _TEST = {
         # From http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/
 
     _TEST = {
         # From http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/

-        'url': 'http://embed.sendtonews.com/player/embed.php?SK=GxfCe0Zo7D&MK=175909&PK=5588&autoplay=on&sound=yes',
+        'url': 'http://embed.sendtonews.com/player2/embedplayer.php?SC=GxfCe0Zo7D-175909-5588&type=single&autoplay=on&sound=YES',

         'info_dict': {
         'info_dict': {

-            'id': 'GxfCe0Zo7D-175909-5588',
-            'ext': 'mp4',
-            'title': 'Recap: CLE 15, CIN 6',
-            'description': '5/16/16: Indians\' bats explode for 15 runs in a win',
-            'duration': 49,
+            'id': 'GxfCe0Zo7D-175909-5588'

         },
         },

+        'playlist_count': 8,
+        # test the first video only to prevent lengthy tests
+        'playlist': [{
+            'info_dict': {
+                'id': '240385',
+                'ext': 'mp4',
+                'title': 'Indians introduce Encarnacion',
+                'description': 'Indians president of baseball operations Chris Antonetti and Edwin Encarnacion discuss the slugger\'s three-year contract with Cleveland',
+                'duration': 137.898,
+                'thumbnail': r're:https?://.*\.jpg$',
+                'upload_date': '20170105',
+                'timestamp': 1483649762,
+            },
+        }],

         'params': {
             # m3u8 download
             'skip_download': True,
         },
     }
 
         'params': {
             # m3u8 download
             'skip_download': True,
         },
     }
 

-    _URL_TEMPLATE = '//embed.sendtonews.com/player/embed.php?SK=%s&MK=%s&PK=%s'
+    _URL_TEMPLATE = '//embed.sendtonews.com/player2/embedplayer.php?SC=%s'

 
     @classmethod
     def _extract_url(cls, webpage):
 
     @classmethod
     def _extract_url(cls, webpage):


@@ -39,48 +52,54 @@ class SendtoNewsIE(JWPlatformBaseIE):
                 .*\bSC=(?P<SC>[0-9a-zA-Z-]+).*
             \1>''', webpage)
         if mobj:
                 .*\bSC=(?P<SC>[0-9a-zA-Z-]+).*
             \1>''', webpage)
         if mobj:

-            sk, mk, pk = mobj.group('SC').split('-')
-            return cls._URL_TEMPLATE % (sk, mk, pk)
+            sc = mobj.group('SC')
+            return cls._URL_TEMPLATE % sc

 
     def _real_extract(self, url):
 
     def _real_extract(self, url):

-        mobj = re.match(self._VALID_URL, url)
-        params = compat_parse_qs(mobj.group('query'))
-
-        if 'SK' not in params or 'MK' not in params or 'PK' not in params:
-            raise ExtractorError('Invalid URL', expected=True)
-
-        video_id = '-'.join([params['SK'][0], params['MK'][0], params['PK'][0]])
-
-        webpage = self._download_webpage(url, video_id)
-
-        jwplayer_data_str = self._search_regex(
-            r'jwplayer\("[^"]+"\)\.setup\((.+?)\);', webpage, 'JWPlayer data')
-        js_vars = {
-            'w': 1024,
-            'h': 768,
-            'modeVar': 'html5',
-        }
-        for name, val in js_vars.items():
-            js_val = '%d' % val if isinstance(val, int) else '"%s"' % val
-            jwplayer_data_str = jwplayer_data_str.replace(':%s,' % name, ':%s,' % js_val)
-
-        info_dict = self._parse_jwplayer_data(
-            self._parse_json(jwplayer_data_str, video_id),
-            video_id, require_title=False, rtmp_params={'no_resume': True})
-
-        title = self._html_search_regex(
-            r'<div[^>]+class="embedTitle">([^<]+)</div>', webpage, 'title')
-        description = self._html_search_regex(
-            r'<div[^>]+class="embedSubTitle">([^<]+)</div>', webpage,
-            'description', fatal=False)
-        duration = parse_duration(self._html_search_regex(
-            r'<div[^>]+class="embedDetails">([0-9:]+)', webpage,
-            'duration', fatal=False))
-
-        info_dict.update({
-            'title': title,
-            'description': description,
-            'duration': duration,
-        })
-
-        return info_dict
+        playlist_id = self._match_id(url)
+
+        data_url = update_url_query(
+            url.replace('embedplayer.php', 'data_read.php'),
+            {'cmd': 'loadInitial'})
+        playlist_data = self._download_json(data_url, playlist_id)
+
+        entries = []
+        for video in playlist_data['playlistData'][0]:
+            info_dict = self._parse_jwplayer_data(
+                video['jwconfiguration'],
+                require_title=False, m3u8_id='hls', rtmp_params={'no_resume': True})
+
+            for f in info_dict['formats']:
+                if f.get('tbr'):
+                    continue
+                tbr = int_or_none(self._search_regex(
+                    r'/(\d+)k/', f['url'], 'bitrate', default=None))
+                if not tbr:
+                    continue
+                f.update({
+                    'format_id': '%s-%d' % (determine_protocol(f), tbr),
+                    'tbr': tbr,
+                })
+            self._sort_formats(info_dict['formats'], ('tbr', 'height', 'width', 'format_id'))
+
+            thumbnails = []
+            if video.get('thumbnailUrl'):
+                thumbnails.append({
+                    'id': 'normal',
+                    'url': video['thumbnailUrl'],
+                })
+            if video.get('smThumbnailUrl'):
+                thumbnails.append({
+                    'id': 'small',
+                    'url': video['smThumbnailUrl'],
+                })
+            info_dict.update({
+                'title': video['S_headLine'].strip(),
+                'description': unescapeHTML(video.get('S_fullStory')),
+                'thumbnails': thumbnails,
+                'duration': float_or_none(video.get('SM_length')),
+                'timestamp': parse_iso8601(video.get('S_sysDate'), delimiter=' '),
+            })
+            entries.append(info_dict)
+
+        return self.playlist_result(entries, playlist_id)