debian/copyright: Update my copyright years.

[youtubedl] / youtube_dl / extractor / brightcove.py

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0ed59bcbc44ecaf413d7003c4f6f605b1201ca10..58ec5c979e0bc88c98dfafa1ffff81b29af264d8 100644 (file)
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -1,8 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
 # coding: utf-8
 from __future__ import unicode_literals
 

-import re
+import base64

 import json
 import json

+import re
+import struct

 
 from .common import InfoExtractor
 from .adobepass import AdobePassIE
 
 from .common import InfoExtractor
 from .adobepass import AdobePassIE


@@ -124,7 +126,7 @@ class BrightcoveLegacyIE(InfoExtractor):
             'playlist_mincount': 7,
         },
         {
             'playlist_mincount': 7,
         },
         {

-            # playlist with 'playlistTab' (https://github.com/rg3/youtube-dl/issues/9965)
+            # playlist with 'playlistTab' (https://github.com/ytdl-org/youtube-dl/issues/9965)

             'url': 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=AQ%7E%7E,AAABXlLMdok%7E,NJ4EoMlZ4rZdx9eU1rkMVd8EaYPBBUlg',
             'info_dict': {
                 'id': '1522758701001',
             'url': 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=AQ%7E%7E,AAABXlLMdok%7E,NJ4EoMlZ4rZdx9eU1rkMVd8EaYPBBUlg',
             'info_dict': {
                 'id': '1522758701001',


@@ -153,10 +155,10 @@ class BrightcoveLegacyIE(InfoExtractor):
         <object class="BrightcoveExperience">{params}</object>
         """
 
         <object class="BrightcoveExperience">{params}</object>
         """
 

-        # Fix up some stupid HTML, see https://github.com/rg3/youtube-dl/issues/1553
+        # Fix up some stupid HTML, see https://github.com/ytdl-org/youtube-dl/issues/1553

         object_str = re.sub(r'(<param(?:\s+[a-zA-Z0-9_]+="[^"]*")*)>',
                             lambda m: m.group(1) + '/>', object_str)
         object_str = re.sub(r'(<param(?:\s+[a-zA-Z0-9_]+="[^"]*")*)>',
                             lambda m: m.group(1) + '/>', object_str)

-        # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
+        # Fix up some stupid XML, see https://github.com/ytdl-org/youtube-dl/issues/1608

         object_str = object_str.replace('<--', '<!--')
         # remove namespace to simplify extraction
         object_str = re.sub(r'(<object[^>]*)(xmlns=".*?")', r'\1', object_str)
         object_str = object_str.replace('<--', '<!--')
         # remove namespace to simplify extraction
         object_str = re.sub(r'(<object[^>]*)(xmlns=".*?")', r'\1', object_str)


@@ -310,6 +312,10 @@ class BrightcoveLegacyIE(InfoExtractor):
                 'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?',
                 expected=True)
 
                 'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?',
                 expected=True)
 

+    def _brightcove_new_url_result(self, publisher_id, video_id):
+        brightcove_new_url = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s' % (publisher_id, video_id)
+        return self.url_result(brightcove_new_url, BrightcoveNewIE.ie_key(), video_id)
+

     def _get_video_info(self, video_id, query, referer=None):
         headers = {}
         linkBase = query.get('linkBaseURL')
     def _get_video_info(self, video_id, query, referer=None):
         headers = {}
         linkBase = query.get('linkBaseURL')


@@ -323,6 +329,28 @@ class BrightcoveLegacyIE(InfoExtractor):
             r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
             'error message', default=None)
         if error_msg is not None:
             r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
             'error message', default=None)
         if error_msg is not None:

+            publisher_id = query.get('publisherId')
+            if publisher_id and publisher_id[0].isdigit():
+                publisher_id = publisher_id[0]
+            if not publisher_id:
+                player_key = query.get('playerKey')
+                if player_key and ',' in player_key[0]:
+                    player_key = player_key[0]
+                else:
+                    player_id = query.get('playerID')
+                    if player_id and player_id[0].isdigit():
+                        player_page = self._download_webpage(
+                            'http://link.brightcove.com/services/player/bcpid' + player_id[0],
+                            video_id, headers=headers, fatal=False)
+                        if player_page:
+                            player_key = self._search_regex(
+                                r'<param\s+name="playerKey"\s+value="([\w~,-]+)"',
+                                player_page, 'player key', fatal=False)
+                if player_key:
+                    enc_pub_id = player_key.split(',')[1].replace('~', '=')
+                    publisher_id = struct.unpack('>Q', base64.urlsafe_b64decode(enc_pub_id))[0]
+                if publisher_id:
+                    return self._brightcove_new_url_result(publisher_id, video_id)

             raise ExtractorError(
                 'brightcove said: %s' % error_msg, expected=True)
 
             raise ExtractorError(
                 'brightcove said: %s' % error_msg, expected=True)
 


@@ -444,14 +472,18 @@ class BrightcoveLegacyIE(InfoExtractor):
                 else:
                     return ad_info
 
                 else:
                     return ad_info
 

-        if 'url' not in info and not info.get('formats'):
-            raise ExtractorError('Unable to extract video url for %s' % video_id)
+        if not info.get('url') and not info.get('formats'):
+            uploader_id = info.get('uploader_id')
+            if uploader_id:
+                info.update(self._brightcove_new_url_result(uploader_id, video_id))
+            else:
+                raise ExtractorError('Unable to extract video url for %s' % video_id)

         return info
 
 
 class BrightcoveNewIE(AdobePassIE):
     IE_NAME = 'brightcove:new'
         return info
 
 
 class BrightcoveNewIE(AdobePassIE):
     IE_NAME = 'brightcove:new'

-    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+|ref:[^&]+)'
+    _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*(?P<content_type>video|playlist)Id=(?P<video_id>\d+|ref:[^&]+)'

     _TESTS = [{
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',
     _TESTS = [{
         'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
         'md5': 'c8100925723840d4b0d243f7025703be',


@@ -464,7 +496,7 @@ class BrightcoveNewIE(AdobePassIE):
             'timestamp': 1441391203,
             'upload_date': '20150904',
             'uploader_id': '929656772001',
             'timestamp': 1441391203,
             'upload_date': '20150904',
             'uploader_id': '929656772001',

-            'formats': 'mincount:22',
+            'formats': 'mincount:20',

         },
     }, {
         # with rtmp streams
         },
     }, {
         # with rtmp streams


@@ -478,12 +510,27 @@ class BrightcoveNewIE(AdobePassIE):
             'timestamp': 1433556729,
             'upload_date': '20150606',
             'uploader_id': '4036320279001',
             'timestamp': 1433556729,
             'upload_date': '20150606',
             'uploader_id': '4036320279001',

-            'formats': 'mincount:41',
+            'formats': 'mincount:39',

         },
         'params': {
             # m3u8 download
             'skip_download': True,
         }
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         }

+    }, {
+        # playlist stream
+        'url': 'https://players.brightcove.net/1752604059001/S13cJdUBz_default/index.html?playlistId=5718313430001',
+        'info_dict': {
+            'id': '5718313430001',
+            'title': 'No Audio Playlist',
+        },
+        'playlist_count': 7,
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        }
+    }, {
+        'url': 'http://players.brightcove.net/5690807595001/HyZNerRl7_default/index.html?playlistId=5743160747001',
+        'only_matching': True,

     }, {
         # ref: prefixed video id
         'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442',
     }, {
         # ref: prefixed video id
         'url': 'http://players.brightcove.net/3910869709001/21519b5c-4b3b-4363-accb-bdc8f358f823_default/index.html?videoId=ref:7069442',


@@ -564,59 +611,7 @@ class BrightcoveNewIE(AdobePassIE):
 
         return entries
 
 
         return entries
 

-    def _real_extract(self, url):
-        url, smuggled_data = unsmuggle_url(url, {})
-        self._initialize_geo_bypass(smuggled_data.get('geo_countries'))
-
-        account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
-
-        webpage = self._download_webpage(
-            'http://players.brightcove.net/%s/%s_%s/index.min.js'
-            % (account_id, player_id, embed), video_id)
-
-        policy_key = None
-
-        catalog = self._search_regex(
-            r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
-        if catalog:
-            catalog = self._parse_json(
-                js_to_json(catalog), video_id, fatal=False)
-            if catalog:
-                policy_key = catalog.get('policyKey')
-
-        if not policy_key:
-            policy_key = self._search_regex(
-                r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
-                webpage, 'policy key', group='pk')
-
-        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id)
-        try:
-            json_data = self._download_json(api_url, video_id, headers={
-                'Accept': 'application/json;pk=%s' % policy_key
-            })
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
-                message = json_data.get('message') or json_data['error_code']
-                if json_data.get('error_subcode') == 'CLIENT_GEO':
-                    self.raise_geo_restricted(msg=message)
-                raise ExtractorError(message, expected=True)
-            raise
-
-        errors = json_data.get('errors')
-        if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
-            custom_fields = json_data['custom_fields']
-            tve_token = self._extract_mvpd_auth(
-                smuggled_data['source_url'], video_id,
-                custom_fields['bcadobepassrequestorid'],
-                custom_fields['bcadobepassresourceid'])
-            json_data = self._download_json(
-                api_url, video_id, headers={
-                    'Accept': 'application/json;pk=%s' % policy_key
-                }, query={
-                    'tveToken': tve_token,
-                })
-
+    def _parse_brightcove_metadata(self, json_data, video_id, headers={}):

         title = json_data['name'].strip()
 
         formats = []
         title = json_data['name'].strip()
 
         formats = []


@@ -624,7 +619,8 @@ class BrightcoveNewIE(AdobePassIE):
             container = source.get('container')
             ext = mimetype2ext(source.get('type'))
             src = source.get('src')
             container = source.get('container')
             ext = mimetype2ext(source.get('type'))
             src = source.get('src')

-            if ext == 'ism' or container == 'WVM':
+            # https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object
+            if ext == 'ism' or container == 'WVM' or source.get('key_systems'):

                 continue
             elif ext == 'm3u8' or container == 'M2TS':
                 if not src:
                 continue
             elif ext == 'm3u8' or container == 'M2TS':
                 if not src:


@@ -681,7 +677,16 @@ class BrightcoveNewIE(AdobePassIE):
                         'format_id': build_format_id('rtmp'),
                     })
                 formats.append(f)
                         'format_id': build_format_id('rtmp'),
                     })
                 formats.append(f)

+        if not formats:
+            # for sonyliv.com DRM protected videos
+            s3_source_url = json_data.get('custom_fields', {}).get('s3sourceurl')
+            if s3_source_url:
+                formats.append({
+                    'url': s3_source_url,
+                    'format_id': 'source',
+                })

 
 

+        errors = json_data.get('errors')

         if not formats and errors:
             error = errors[0]
             raise ExtractorError(
         if not formats and errors:
             error = errors[0]
             raise ExtractorError(


@@ -689,6 +694,9 @@ class BrightcoveNewIE(AdobePassIE):
 
         self._sort_formats(formats)
 
 
         self._sort_formats(formats)
 

+        for f in formats:
+            f.setdefault('http_headers', {}).update(headers)
+

         subtitles = {}
         for text_track in json_data.get('text_tracks', []):
             if text_track.get('src'):
         subtitles = {}
         for text_track in json_data.get('text_tracks', []):
             if text_track.get('src'):


@@ -708,9 +716,82 @@ class BrightcoveNewIE(AdobePassIE):
             'thumbnail': json_data.get('thumbnail') or json_data.get('poster'),
             'duration': duration,
             'timestamp': parse_iso8601(json_data.get('published_at')),
             'thumbnail': json_data.get('thumbnail') or json_data.get('poster'),
             'duration': duration,
             'timestamp': parse_iso8601(json_data.get('published_at')),

-            'uploader_id': account_id,
+            'uploader_id': json_data.get('account_id'),

             'formats': formats,
             'subtitles': subtitles,
             'tags': json_data.get('tags', []),
             'is_live': is_live,
         }
             'formats': formats,
             'subtitles': subtitles,
             'tags': json_data.get('tags', []),
             'is_live': is_live,
         }

+
+    def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+        self._initialize_geo_bypass({
+            'countries': smuggled_data.get('geo_countries'),
+            'ip_blocks': smuggled_data.get('geo_ip_blocks'),
+        })
+
+        account_id, player_id, embed, content_type, video_id = re.match(self._VALID_URL, url).groups()
+
+        webpage = self._download_webpage(
+            'http://players.brightcove.net/%s/%s_%s/index.min.js'
+            % (account_id, player_id, embed), video_id)
+
+        policy_key = None
+
+        catalog = self._search_regex(
+            r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
+        if catalog:
+            catalog = self._parse_json(
+                js_to_json(catalog), video_id, fatal=False)
+            if catalog:
+                policy_key = catalog.get('policyKey')
+
+        if not policy_key:
+            policy_key = self._search_regex(
+                r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
+                webpage, 'policy key', group='pk')
+
+        api_url = 'https://edge.api.brightcove.com/playback/v1/accounts/%s/%ss/%s' % (account_id, content_type, video_id)
+        headers = {
+            'Accept': 'application/json;pk=%s' % policy_key,
+        }
+        referrer = smuggled_data.get('referrer')
+        if referrer:
+            headers.update({
+                'Referer': referrer,
+                'Origin': re.search(r'https?://[^/]+', referrer).group(0),
+            })
+        try:
+            json_data = self._download_json(api_url, video_id, headers=headers)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
+                message = json_data.get('message') or json_data['error_code']
+                if json_data.get('error_subcode') == 'CLIENT_GEO':
+                    self.raise_geo_restricted(msg=message)
+                raise ExtractorError(message, expected=True)
+            raise
+
+        errors = json_data.get('errors')
+        if errors and errors[0].get('error_subcode') == 'TVE_AUTH':
+            custom_fields = json_data['custom_fields']
+            tve_token = self._extract_mvpd_auth(
+                smuggled_data['source_url'], video_id,
+                custom_fields['bcadobepassrequestorid'],
+                custom_fields['bcadobepassresourceid'])
+            json_data = self._download_json(
+                api_url, video_id, headers={
+                    'Accept': 'application/json;pk=%s' % policy_key
+                }, query={
+                    'tveToken': tve_token,
+                })
+
+        if content_type == 'playlist':
+            return self.playlist_result(
+                [self._parse_brightcove_metadata(vid, vid.get('id'), headers)
+                 for vid in json_data.get('videos', []) if vid.get('id')],
+                json_data.get('id'), json_data.get('name'),
+                json_data.get('description'))
+
+        return self._parse_brightcove_metadata(
+            json_data, video_id, headers=headers)