Update upstream source from tag 'upstream/2019.01.17'

[youtubedl] / youtube_dl / extractor / brightcove.py

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f04505011c39fbfe315763ce8182daf6c2b9f663..465ae396eb9c690d84dd00a64f7439a74c74ba33 100644 (file)
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -1,8 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
 # coding: utf-8
 from __future__ import unicode_literals
 

-import re
+import base64

 import json
 import json

+import re
+import struct

 
 from .common import InfoExtractor
 from .adobepass import AdobePassIE
 
 from .common import InfoExtractor
 from .adobepass import AdobePassIE


@@ -310,6 +312,10 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?',
                 expected=True)
 
                 'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?',
                 expected=True)
 

+    def _brightcove_new_url_result(self, publisher_id, video_id):
+        brightcove_new_url = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (publisher_id, video_id)
+        return self.url_result(brightcove_new_url, BrightcoveNewIE.ie_key(), video_id)
+

     def _get_video_info(self, video_id, query, referer=None):
         headers = {}
         linkBase = query.get('linkBaseURL')
     def _get_video_info(self, video_id, query, referer=None):
         headers = {}
         linkBase = query.get('linkBaseURL')


@@ -323,6 +329,28 @@ class BrightcoveLegacyIE(InfoExtractor):
             r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
             'error message', default=None)
         if error_msg is not None:
             r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
             'error message', default=None)
         if error_msg is not None:

+            publisher_id = query.get('publisherId')
+            if publisher_id and publisher_id[0].isdigit():
+                publisher_id = publisher_id[0]
+            if not publisher_id:
+                player_key = query.get('playerKey')
+                if player_key and ',' in player_key[0]:
+                    player_key = player_key[0]
+                else:
+                    player_id = query.get('playerID')
+                    if player_id and player_id[0].isdigit():
+                        player_page = self._download_webpage(
+                            'http://link.brightcove.com/services/player/bcpid' + player_id[0],
+                            video_id, headers=headers, fatal=False)
+                        if player_page:
+                            player_key = self._search_regex(
+                                r'<param\s+name="playerKey"\s+value="([\w~,-]+)"',
+                                player_page, 'player key', fatal=False)
+                if player_key:
+                    enc_pub_id = player_key.split(',')[1].replace('~', '=')
+                    publisher_id = struct.unpack('>Q', base64.urlsafe_b64decode(enc_pub_id))[0]
+                if publisher_id:
+                    return self._brightcove_new_url_result(publisher_id, video_id)

             raise ExtractorError(
                 'brightcove said: %s' % error_msg, expected=True)
 
             raise ExtractorError(
                 'brightcove said: %s' % error_msg, expected=True)
 


@@ -444,8 +472,12 @@ class BrightcoveLegacyIE(InfoExtractor):
                 else:
                     return ad_info
 
                 else:
                     return ad_info
 

-        if 'url' not in info and not info.get('formats'):
-            raise ExtractorError('Unable to extract video url for %s' % video_id)
+        if not info.get('url') and not info.get('formats'):
+            uploader_id = info.get('uploader_id')
+            if uploader_id:
+                info.update(self._brightcove_new_url_result(uploader_id, video_id))
+            else:
+                raise ExtractorError('Unable to extract video url for %s' % video_id)

         return info
 
 
         return info
 
 


@@ -564,7 +596,7 @@ class BrightcoveNewIE(AdobePassIE):
 
         return entries
 
 
         return entries
 

-    def _parse_brightcove_metadata(self, json_data, video_id):
+    def _parse_brightcove_metadata(self, json_data, video_id, headers={}):

         title = json_data['name'].strip()
 
         formats = []
         title = json_data['name'].strip()
 
         formats = []


@@ -572,7 +604,8 @@ class BrightcoveNewIE(AdobePassIE):
             container = source.get('container')
             ext = mimetype2ext(source.get('type'))
             src = source.get('src')
             container = source.get('container')
             ext = mimetype2ext(source.get('type'))
             src = source.get('src')

-            if ext == 'ism' or container == 'WVM':
+            # https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object
+            if ext == 'ism' or container == 'WVM' or source.get('key_systems'):

                 continue
             elif ext == 'm3u8' or container == 'M2TS':
                 if not src:
                 continue
             elif ext == 'm3u8' or container == 'M2TS':
                 if not src:


@@ -629,6 +662,14 @@ class BrightcoveNewIE(AdobePassIE):
                         'format_id': build_format_id('rtmp'),
                     })
                 formats.append(f)
                         'format_id': build_format_id('rtmp'),
                     })
                 formats.append(f)

+        if not formats:
+            # for sonyliv.com DRM protected videos
+            s3_source_url = json_data.get('custom_fields', {}).get('s3sourceurl')
+            if s3_source_url:
+                formats.append({
+                    'url': s3_source_url,
+                    'format_id': 'source',
+                })

 
         errors = json_data.get('errors')
         if not formats and errors:
 
         errors = json_data.get('errors')
         if not formats and errors:


@@ -638,6 +679,9 @@ class BrightcoveNewIE(AdobePassIE):
 
         self._sort_formats(formats)
 
 
         self._sort_formats(formats)
 

+        for f in formats:
+            f.setdefault('http_headers', {}).update(headers)
+

         subtitles = {}
         for text_track in json_data.get('text_tracks', []):
             if text_track.get('src'):
         subtitles = {}
         for text_track in json_data.get('text_tracks', []):
             if text_track.get('src'):


@@ -666,7 +710,10 @@ class BrightcoveNewIE(AdobePassIE):
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})

-        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
+        self._initialize_geo_bypass({
+            'countries': smuggled_data.get('geo_countries'),
+            'ip_blocks': smuggled_data.get('geo_ip_blocks'),
+        })

 
         account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
 
 
         account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
 


@@ -690,10 +737,17 @@ class BrightcoveNewIE(AdobePassIE):
                 webpage, 'policy key', group='pk')
 
         api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)
                 webpage, 'policy key', group='pk')
 
         api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)

-        try:
-            json_data = self._download_json(api_url, video_id, headers={
-                'Accept': 'application/json;pk=%s' % policy_key
+        headers = {
+            'Accept': 'application/json;pk=%s' % policy_key,
+        }
+        referrer = smuggled_data.get('referrer')
+        if referrer:
+            headers.update({
+                'Referer': referrer,
+                'Origin': re.search(r'https?://[^/]+', referrer).group(0),

             })
             })

+        try:
+            json_data = self._download_json(api_url, video_id, headers=headers)

         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
                 json_data = self._parse_json(e.cause.read().decode(), video_id)[0]


@@ -717,4 +771,5 @@ class BrightcoveNewIE(AdobePassIE):
                     'tveToken': tve_token,
                 })
 
                     'tveToken': tve_token,
                 })
 

-        return self._parse_brightcove_metadata(json_data, video_id)
+        return self._parse_brightcove_metadata(
+            json_data, video_id, headers=headers)