Update upstream source from tag 'upstream/2018.09.10'

[youtubedl] / youtube_dl / extractor / abc.py

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 60f753b95c6e89158c3d292bd62e8bee2cd74746..4ac323bf6de6d17016c2425c133aad460072cadd 100644 (file)
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -1,6 +1,9 @@
 from __future__ import unicode_literals
 
 from __future__ import unicode_literals
 

+import hashlib
+import hmac

 import re
 import re

+import time

 
 from .common import InfoExtractor
 from ..compat import compat_str
 
 from .common import InfoExtractor
 from ..compat import compat_str


@@ -10,6 +13,8 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     try_get,
     int_or_none,
     parse_iso8601,
     try_get,

+    unescapeHTML,
+    update_url_query,

 )
 
 
 )
 
 


@@ -100,46 +105,59 @@ class ABCIE(InfoExtractor):
 
 class ABCIViewIE(InfoExtractor):
     IE_NAME = 'abc.net.au:iview'
 
 class ABCIViewIE(InfoExtractor):
     IE_NAME = 'abc.net.au:iview'

-    _VALID_URL = r'https?://iview\.abc\.net\.au/programs/[^/]+/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://iview\.abc\.net\.au/(?:[^/]+/)*video/(?P<id>[^/?#]+)'
+    _GEO_COUNTRIES = ['AU']

 
     # ABC iview programs are normally available for 14 days only.
     _TESTS = [{
 
     # ABC iview programs are normally available for 14 days only.
     _TESTS = [{

-        'url': 'http://iview.abc.net.au/programs/diaries-of-a-broken-mind/ZX9735A001S00',
+        'url': 'https://iview.abc.net.au/show/ben-and-hollys-little-kingdom/series/0/video/ZX9371A050S00',

         'md5': 'cde42d728b3b7c2b32b1b94b4a548afc',
         'info_dict': {
         'md5': 'cde42d728b3b7c2b32b1b94b4a548afc',
         'info_dict': {

-            'id': 'ZX9735A001S00',
+            'id': 'ZX9371A050S00',

             'ext': 'mp4',
             'ext': 'mp4',

-            'title': 'Diaries Of A Broken Mind',
-            'description': 'md5:7de3903874b7a1be279fe6b68718fc9e',
-            'upload_date': '20161010',
-            'uploader_id': 'abc2',
-            'timestamp': 1476064920,
+            'title': "Gaston's Birthday",
+            'series': "Ben And Holly's Little Kingdom",
+            'description': 'md5:f9de914d02f226968f598ac76f105bcf',
+            'upload_date': '20180604',
+            'uploader_id': 'abc4kids',
+            'timestamp': 1528140219,
+        },
+        'params': {
+            'skip_download': True,

         },
         },

-        'skip': 'Video gone',

     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

-        webpage = self._download_webpage(url, video_id)
-        video_params = self._parse_json(self._search_regex(
-            r'videoParams\s*=\s*({.+?});', webpage, 'video params'), video_id)
-        title = video_params.get('title') or video_params['seriesTitle']
-        stream = next(s for s in video_params['playlist'] if s.get('type') == 'program')
-
-        format_urls = [
-            try_get(stream, lambda x: x['hds-unmetered'], compat_str)]
-
-        # May have higher quality video
-        sd_url = try_get(
-            stream, lambda x: x['streams']['hds']['sd'], compat_str)
-        if sd_url:
-            format_urls.append(sd_url.replace('metered', 'um'))
-
-        formats = []
-        for format_url in format_urls:
-            if format_url:
-                formats.extend(
-                    self._extract_akamai_formats(format_url, video_id))
+        video_params = self._download_json(
+            'https://iview.abc.net.au/api/programs/' + video_id, video_id)
+        title = unescapeHTML(video_params.get('title') or video_params['seriesTitle'])
+        stream = next(s for s in video_params['playlist'] if s.get('type') in ('program', 'livestream'))
+
+        house_number = video_params.get('episodeHouseNumber') or video_id
+        path = '/auth/hls/sign?ts={0}&hn={1}&d=android-tablet'.format(
+            int(time.time()), house_number)
+        sig = hmac.new(
+            b'android.content.res.Resources',
+            path.encode('utf-8'), hashlib.sha256).hexdigest()
+        token = self._download_webpage(
+            'http://iview.abc.net.au{0}&sig={1}'.format(path, sig), video_id)
+
+        def tokenize_url(url, token):
+            return update_url_query(url, {
+                'hdnea': token,
+            })
+
+        for sd in ('sd', 'sd-low'):
+            sd_url = try_get(
+                stream, lambda x: x['streams']['hls'][sd], compat_str)
+            if not sd_url:
+                continue
+            formats = self._extract_m3u8_formats(
+                tokenize_url(sd_url, token), video_id, 'mp4',
+                entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+            if formats:
+                break

         self._sort_formats(formats)
 
         subtitles = {}
         self._sort_formats(formats)
 
         subtitles = {}


@@ -150,18 +168,26 @@ class ABCIViewIE(InfoExtractor):
                 'ext': 'vtt',
             }]
 
                 'ext': 'vtt',
             }]
 

+        is_live = video_params.get('livestream') == '1'
+        if is_live:
+            title = self._live_title(title)
+

         return {
             'id': video_id,
             'title': title,
         return {
             'id': video_id,
             'title': title,

-            'description': self._html_search_meta(['og:description', 'twitter:description'], webpage),
-            'thumbnail': self._html_search_meta(['og:image', 'twitter:image:src'], webpage),
+            'description': video_params.get('description'),
+            'thumbnail': video_params.get('thumbnail'),

             'duration': int_or_none(video_params.get('eventDuration')),
             'timestamp': parse_iso8601(video_params.get('pubDate'), ' '),
             'duration': int_or_none(video_params.get('eventDuration')),
             'timestamp': parse_iso8601(video_params.get('pubDate'), ' '),

-            'series': video_params.get('seriesTitle'),
+            'series': unescapeHTML(video_params.get('seriesTitle')),

             'series_id': video_params.get('seriesHouseNumber') or video_id[:7],
             'series_id': video_params.get('seriesHouseNumber') or video_id[:7],

-            'episode_number': int_or_none(self._html_search_meta('episodeNumber', webpage, default=None)),
-            'episode': self._html_search_meta('episode_title', webpage, default=None),
+            'season_number': int_or_none(self._search_regex(
+                r'\bSeries\s+(\d+)\b', title, 'season number', default=None)),
+            'episode_number': int_or_none(self._search_regex(
+                r'\bEp\s+(\d+)\b', title, 'episode number', default=None)),
+            'episode_id': house_number,

             'uploader_id': video_params.get('channel'),
             'formats': formats,
             'subtitles': subtitles,
             'uploader_id': video_params.get('channel'),
             'formats': formats,
             'subtitles': subtitles,

+            'is_live': is_live,

         }
         }