Update upstream source from tag 'upstream/2020.06.16'

[youtubedl] / youtube_dl / extractor / discovery.py

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index b70c307a75b520aa63e44fea69762cacc6bbc319..e0139cc862d74bc3c20a9d6567747b96b642c730 100644 (file)
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -5,31 +5,24 @@ import re
 import string
 
 from .discoverygo import DiscoveryGoBaseIE
 import string
 
 from .discoverygo import DiscoveryGoBaseIE

-from ..compat import (
-    compat_str,
-    compat_urllib_parse_unquote,
-)
-from ..utils import (
-    ExtractorError,
-    try_get,
-)
+from ..compat import compat_urllib_parse_unquote
+from ..utils import ExtractorError

 from ..compat import compat_HTTPError
 
 
 class DiscoveryIE(DiscoveryGoBaseIE):
     _VALID_URL = r'''(?x)https?://
         (?P<site>
 from ..compat import compat_HTTPError
 
 
 class DiscoveryIE(DiscoveryGoBaseIE):
     _VALID_URL = r'''(?x)https?://
         (?P<site>

-            (?:www\.)?
+            go\.discovery|
+            www\.

                 (?:
                 (?:

-                    discovery|

                     investigationdiscovery|
                     discoverylife|
                     animalplanet|
                     ahctv|
                     destinationamerica|
                     sciencechannel|
                     investigationdiscovery|
                     discoverylife|
                     animalplanet|
                     ahctv|
                     destinationamerica|
                     sciencechannel|

-                    tlc|
-                    velocity
+                    tlc

                 )|
             watch\.
                 (?:
                 )|
             watch\.
                 (?:


@@ -40,15 +33,15 @@ class DiscoveryIE(DiscoveryGoBaseIE):
                     cookingchanneltv|
                     motortrend
                 )
                     cookingchanneltv|
                     motortrend
                 )

-        )\.com(?P<path>/tv-shows/[^/]+/(?:video|full-episode)s/(?P<id>[^./?#]+))'''
+        )\.com/tv-shows/(?P<show_slug>[^/]+)/(?:video|full-episode)s/(?P<id>[^./?#]+)'''

     _TESTS = [{
     _TESTS = [{

-        'url': 'https://www.discovery.com/tv-shows/cash-cab/videos/dave-foley',
+        'url': 'https://go.discovery.com/tv-shows/cash-cab/videos/riding-with-matthew-perry',

         'info_dict': {
         'info_dict': {

-            'id': '5a2d9b4d6b66d17a5026e1fd',
+            'id': '5a2f35ce6b66d17a5026e29e',

             'ext': 'mp4',
             'ext': 'mp4',

-            'title': 'Dave Foley',
-            'description': 'md5:4b39bcafccf9167ca42810eb5f28b01f',
-            'duration': 608,
+            'title': 'Riding with Matthew Perry',
+            'description': 'md5:a34333153e79bc4526019a5129e7f878',
+            'duration': 84,

         },
         'params': {
             'skip_download': True,  # requires ffmpeg
         },
         'params': {
             'skip_download': True,  # requires ffmpeg


@@ -56,20 +49,20 @@ class DiscoveryIE(DiscoveryGoBaseIE):
     }, {
         'url': 'https://www.investigationdiscovery.com/tv-shows/final-vision/full-episodes/final-vision',
         'only_matching': True,
     }, {
         'url': 'https://www.investigationdiscovery.com/tv-shows/final-vision/full-episodes/final-vision',
         'only_matching': True,

+    }, {
+        'url': 'https://go.discovery.com/tv-shows/alaskan-bush-people/videos/follow-your-own-road',
+        'only_matching': True,
+    }, {
+        # using `show_slug` is important to get the correct video data
+        'url': 'https://www.sciencechannel.com/tv-shows/mythbusters-on-science/full-episodes/christmas-special',
+        'only_matching': True,

     }]
     _GEO_COUNTRIES = ['US']
     _GEO_BYPASS = False
     }]
     _GEO_COUNTRIES = ['US']
     _GEO_BYPASS = False

+    _API_BASE_URL = 'https://api.discovery.com/v1/'

 
     def _real_extract(self, url):
 
     def _real_extract(self, url):

-        site, path, display_id = re.match(self._VALID_URL, url).groups()
-        webpage = self._download_webpage(url, display_id)
-
-        react_data = self._parse_json(self._search_regex(
-            r'window\.__reactTransmitPacket\s*=\s*({.+?});',
-            webpage, 'react data'), display_id)
-        content_blocks = react_data['layout'][path]['contentBlocks']
-        video = next(cb for cb in content_blocks if cb.get('type') == 'video')['content']['items'][0]
-        video_id = video['id']
+        site, show_slug, display_id = re.match(self._VALID_URL, url).groups()

 
         access_token = None
         cookies = self._get_cookies(url)
 
         access_token = None
         cookies = self._get_cookies(url)


@@ -79,27 +72,36 @@ class DiscoveryIE(DiscoveryGoBaseIE):
         if auth_storage_cookie and auth_storage_cookie.value:
             auth_storage = self._parse_json(compat_urllib_parse_unquote(
                 compat_urllib_parse_unquote(auth_storage_cookie.value)),
         if auth_storage_cookie and auth_storage_cookie.value:
             auth_storage = self._parse_json(compat_urllib_parse_unquote(
                 compat_urllib_parse_unquote(auth_storage_cookie.value)),

-                video_id, fatal=False) or {}
+                display_id, fatal=False) or {}

             access_token = auth_storage.get('a') or auth_storage.get('access_token')
 
         if not access_token:
             access_token = self._download_json(
             access_token = auth_storage.get('a') or auth_storage.get('access_token')
 
         if not access_token:
             access_token = self._download_json(

-                'https://%s.com/anonymous' % site, display_id, query={
+                'https://%s.com/anonymous' % site, display_id,
+                'Downloading token JSON metadata', query={

                     'authRel': 'authorization',
                     'authRel': 'authorization',

-                    'client_id': try_get(
-                        react_data, lambda x: x['application']['apiClientId'],
-                        compat_str) or '3020a40c2356a645b4b4',
+                    'client_id': '3020a40c2356a645b4b4',

                     'nonce': ''.join([random.choice(string.ascii_letters) for _ in range(32)]),
                     'nonce': ''.join([random.choice(string.ascii_letters) for _ in range(32)]),

-                    'redirectUri': 'https://fusion.ddmcdn.com/app/mercury-sdk/180/redirectHandler.html?https://www.%s.com' % site,
+                    'redirectUri': 'https://www.discovery.com/',

                 })['access_token']
 
                 })['access_token']
 

-        try:
-            headers = self.geo_verification_headers()
-            headers['Authorization'] = 'Bearer ' + access_token
+        headers = self.geo_verification_headers()
+        headers['Authorization'] = 'Bearer ' + access_token

 
 

+        try:
+            video = self._download_json(
+                self._API_BASE_URL + 'content/videos',
+                display_id, 'Downloading content JSON metadata',
+                headers=headers, query={
+                    'embed': 'show.name',
+                    'fields': 'authenticated,description.detailed,duration,episodeNumber,id,name,parental.rating,season.number,show,tags',
+                    'slug': display_id,
+                    'show_slug': show_slug,
+                })[0]
+            video_id = video['id']

             stream = self._download_json(
             stream = self._download_json(

-                'https://api.discovery.com/v1/streaming/video/' + video_id,
-                display_id, headers=headers)
+                self._API_BASE_URL + 'streaming/video/' + video_id,
+                display_id, 'Downloading streaming JSON metadata', headers=headers)

         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
                 e_description = self._parse_json(
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
                 e_description = self._parse_json(