Update upstream source from tag 'upstream/2018.03.14'

[youtubedl] / youtube_dl / extractor / atresplayer.py

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index f016368fa8d0890de874a774b2a4a18db60a01c6..1a31ebe0820d67df4b768e8102a134b34cfff0c5 100644 (file)
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -2,23 +2,24 @@ from __future__ import unicode_literals
 
 import time
 import hmac
 
 import time
 import hmac

+import hashlib
+import re

 
 

-from .subtitles import SubtitlesInfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse,
-    compat_urllib_request,
-)
+from .common import InfoExtractor
+from ..compat import compat_str

 from ..utils import (
 from ..utils import (

-    int_or_none,
+    ExtractorError,

     float_or_none,
     float_or_none,

+    int_or_none,
+    sanitized_Request,
+    urlencode_postdata,

     xpath_text,
     xpath_text,

-    ExtractorError,

 )
 
 
 )
 
 

-class AtresPlayerIE(SubtitlesInfoExtractor):
+class AtresPlayerIE(InfoExtractor):

     _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
     _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'

+    _NETRC_MACHINE = 'atresplayer'

     _TESTS = [
         {
             'url': 'http://www.atresplayer.com/television/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_2014122100174.html',
     _TESTS = [
         {
             'url': 'http://www.atresplayer.com/television/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_2014122100174.html',


@@ -29,7 +30,20 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
                 'title': 'Especial Solidario de Nochebuena',
                 'description': 'md5:e2d52ff12214fa937107d21064075bf1',
                 'duration': 5527.6,
                 'title': 'Especial Solidario de Nochebuena',
                 'description': 'md5:e2d52ff12214fa937107d21064075bf1',
                 'duration': 5527.6,

-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
+            },
+            'skip': 'This video is only available for registered users'
+        },
+        {
+            'url': 'http://www.atresplayer.com/television/especial/videoencuentros/temporada-1/capitulo-112-david-bustamante_2014121600375.html',
+            'md5': '6e52cbb513c405e403dbacb7aacf8747',
+            'info_dict': {
+                'id': 'capitulo-112-david-bustamante',
+                'ext': 'flv',
+                'title': 'David Bustamante',
+                'description': 'md5:f33f1c0a05be57f6708d4dd83a3b81c6',
+                'duration': 1439.0,
+                'thumbnail': r're:^https?://.*\.jpg$',

             },
         },
         {
             },
         },
         {


@@ -49,6 +63,13 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
 
     _LOGIN_URL = 'https://servicios.atresplayer.com/j_spring_security_check'
 
 
     _LOGIN_URL = 'https://servicios.atresplayer.com/j_spring_security_check'
 

+    _ERRORS = {
+        'UNPUBLISHED': 'We\'re sorry, but this video is not yet available.',
+        'DELETED': 'This video has expired and is no longer available for online streaming.',
+        'GEOUNPUBLISHED': 'We\'re sorry, but this video is not available in your region due to right restrictions.',
+        # 'PREMIUM': 'PREMIUM',
+    }
+

     def _real_initialize(self):
         self._login()
 
     def _real_initialize(self):
         self._login()
 


@@ -62,14 +83,15 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
             'j_password': password,
         }
 
             'j_password': password,
         }
 

-        request = compat_urllib_request.Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        request = sanitized_Request(
+            self._LOGIN_URL, urlencode_postdata(login_form))

         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         response = self._download_webpage(
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         response = self._download_webpage(

-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')

 
         error = self._html_search_regex(
 
         error = self._html_search_regex(

-            r'(?s)<ul class="list_error">(.+?)</ul>', response, 'error', default=None)
+            r'(?s)<ul[^>]+class="[^"]*\blist_error\b[^"]*">(.+?)</ul>',
+            response, 'error', default=None)

         if error:
             raise ExtractorError(
                 'Unable to login: %s' % error, expected=True)
         if error:
             raise ExtractorError(
                 'Unable to login: %s' % error, expected=True)


@@ -82,58 +104,76 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
         episode_id = self._search_regex(
             r'episode="([^"]+)"', webpage, 'episode id')
 
         episode_id = self._search_regex(
             r'episode="([^"]+)"', webpage, 'episode id')
 

+        request = sanitized_Request(
+            self._PLAYER_URL_TEMPLATE % episode_id,
+            headers={'User-Agent': self._USER_AGENT})
+        player = self._download_json(request, episode_id, 'Downloading player JSON')
+
+        episode_type = player.get('typeOfEpisode')
+        error_message = self._ERRORS.get(episode_type)
+        if error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
+
+        formats = []
+        video_url = player.get('urlVideo')
+        if video_url:
+            format_info = {
+                'url': video_url,
+                'format_id': 'http',
+            }
+            mobj = re.search(r'(?P<bitrate>\d+)K_(?P<width>\d+)x(?P<height>\d+)', video_url)
+            if mobj:
+                format_info.update({
+                    'width': int_or_none(mobj.group('width')),
+                    'height': int_or_none(mobj.group('height')),
+                    'tbr': int_or_none(mobj.group('bitrate')),
+                })
+            formats.append(format_info)
+

         timestamp = int_or_none(self._download_webpage(
             self._TIME_API_URL,
             video_id, 'Downloading timestamp', fatal=False), 1000, time.time())
         timestamp_shifted = compat_str(timestamp + self._TIMESTAMP_SHIFT)
         token = hmac.new(
             self._MAGIC.encode('ascii'),
         timestamp = int_or_none(self._download_webpage(
             self._TIME_API_URL,
             video_id, 'Downloading timestamp', fatal=False), 1000, time.time())
         timestamp_shifted = compat_str(timestamp + self._TIMESTAMP_SHIFT)
         token = hmac.new(
             self._MAGIC.encode('ascii'),

-            (episode_id + timestamp_shifted).encode('utf-8')
+            (episode_id + timestamp_shifted).encode('utf-8'), hashlib.md5

         ).hexdigest()
 
         ).hexdigest()
 

-        formats = []
-        for fmt in ['windows', 'android_tablet']:
-            request = compat_urllib_request.Request(
-                self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
-            request.add_header('User-Agent', self._USER_AGENT)
-
-            fmt_json = self._download_json(
-                request, video_id, 'Downloading %s video JSON' % fmt)
-
-            result = fmt_json.get('resultDes')
-            if result.lower() != 'ok':
-                raise ExtractorError(
-                    '%s returned error: %s' % (self.IE_NAME, result), expected=True)
-
-            for format_id, video_url in fmt_json['resultObject'].items():
-                if format_id == 'token' or not video_url.startswith('http'):
-                    continue
-                if video_url.endswith('/Manifest'):
-                    if 'geodeswowsmpra3player' in video_url:
-                        f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
-                        f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
-                        # this videos are protected by DRM, the f4m downloader doesn't support them
-                        continue
-                    else:
-                        f4m_url = video_url[:-9] + '/manifest.f4m'
-                    formats.extend(self._extract_f4m_formats(f4m_url, video_id))
-                else:
-                    formats.append({
-                        'url': video_url,
-                        'format_id': 'android-%s' % format_id,
-                        'preference': 1,
-                    })
-        self._sort_formats(formats)
+        request = sanitized_Request(
+            self._URL_VIDEO_TEMPLATE.format('windows', episode_id, timestamp_shifted, token),
+            headers={'User-Agent': self._USER_AGENT})

 
 

-        player = self._download_json(
-            self._PLAYER_URL_TEMPLATE % episode_id,
-            episode_id)
+        fmt_json = self._download_json(
+            request, video_id, 'Downloading windows video JSON')
+
+        result = fmt_json.get('resultDes')
+        if result.lower() != 'ok':
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, result), expected=True)
+
+        for format_id, video_url in fmt_json['resultObject'].items():
+            if format_id == 'token' or not video_url.startswith('http'):
+                continue
+            if 'geodeswowsmpra3player' in video_url:
+                # f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
+                # f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
+                # this videos are protected by DRM, the f4m downloader doesn't support them
+                continue
+            video_url_hd = video_url.replace('free_es', 'es')
+            formats.extend(self._extract_f4m_formats(
+                video_url_hd[:-9] + '/manifest.f4m', video_id, f4m_id='hds',
+                fatal=False))
+            formats.extend(self._extract_mpd_formats(
+                video_url_hd[:-9] + '/manifest.mpd', video_id, mpd_id='dash',
+                fatal=False))
+        self._sort_formats(formats)

 
         path_data = player.get('pathData')
 
         episode = self._download_xml(
 
         path_data = player.get('pathData')
 
         episode = self._download_xml(

-            self._EPISODE_URL_TEMPLATE % path_data,
-            video_id, 'Downloading episode XML')
+            self._EPISODE_URL_TEMPLATE % path_data, video_id,
+            'Downloading episode XML')

 
         duration = float_or_none(xpath_text(
             episode, './media/asset/info/technical/contentDuration', 'duration'))
 
         duration = float_or_none(xpath_text(
             episode, './media/asset/info/technical/contentDuration', 'duration'))


@@ -144,13 +184,12 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
         thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')
 
         subtitles = {}
         thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')
 
         subtitles = {}

-        subtitle = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
-        if subtitle:
-            subtitles['es'] = subtitle
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+        subtitle_url = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
+        if subtitle_url:
+            subtitles['es'] = [{
+                'ext': 'srt',
+                'url': subtitle_url,
+            }]

 
         return {
             'id': video_id,
 
         return {
             'id': video_id,


@@ -159,5 +198,5 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,

-            'subtitles': self.extract_subtitles(video_id, subtitles),
+            'subtitles': subtitles,

         }
         }