Merge tag 'upstream/2017.03.07'

[youtubedl] / youtube_dl / extractor / acast.py

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index 92eee8119da420a78782d5eea05a2d3564468d3a..6dace305141423ec35d25abe3c4283476a762b2f 100644 (file)
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -2,45 +2,60 @@
 from __future__ import unicode_literals
 
 import re
 from __future__ import unicode_literals
 
 import re

+import functools

 
 from .common import InfoExtractor
 from ..compat import compat_str
 
 from .common import InfoExtractor
 from ..compat import compat_str

-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    OnDemandPagedList,
+)

 
 
 class ACastIE(InfoExtractor):
     IE_NAME = 'acast'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'
 
 
 class ACastIE(InfoExtractor):
     IE_NAME = 'acast'
     _VALID_URL = r'https?://(?:www\.)?acast\.com/(?P<channel>[^/]+)/(?P<id>[^/#?]+)'

-    _TEST = {
+    _TESTS = [{
+        # test with one bling

         'url': 'https://www.acast.com/condenasttraveler/-where-are-you-taipei-101-taiwan',
         'md5': 'ada3de5a1e3a2a381327d749854788bb',
         'info_dict': {
             'id': '57de3baa-4bb0-487e-9418-2692c1277a34',
             'ext': 'mp3',
             'title': '"Where Are You?": Taipei 101, Taiwan',
         'url': 'https://www.acast.com/condenasttraveler/-where-are-you-taipei-101-taiwan',
         'md5': 'ada3de5a1e3a2a381327d749854788bb',
         'info_dict': {
             'id': '57de3baa-4bb0-487e-9418-2692c1277a34',
             'ext': 'mp3',
             'title': '"Where Are You?": Taipei 101, Taiwan',

-            'timestamp': 1196172000000,
+            'timestamp': 1196172000,
+            'upload_date': '20071127',

             'description': 'md5:a0b4ef3634e63866b542e5b1199a1a0e',
             'duration': 211,
         }
             'description': 'md5:a0b4ef3634e63866b542e5b1199a1a0e',
             'duration': 211,
         }

-    }
+    }, {
+        # test with multiple blings
+        'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
+        'md5': '55c0097badd7095f494c99a172f86501',
+        'info_dict': {
+            'id': '2a92b283-1a75-4ad8-8396-499c641de0d9',
+            'ext': 'mp3',
+            'title': '2. Raggarmordet - Röster ur det förflutna',
+            'timestamp': 1477346700,
+            'upload_date': '20161024',
+            'description': 'md5:4f81f6d8cf2e12ee21a321d8bca32db4',
+            'duration': 2797,
+        }
+    }]

 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()
 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()

-
-        embed_page = self._download_webpage(
-            re.sub('(?:www\.)?acast\.com', 'embedcdn.acast.com', url), display_id)
-        cast_data = self._parse_json(self._search_regex(
-            r'window\[\'acast/queries\'\]\s*=\s*([^;]+);', embed_page, 'acast data'),
-            display_id)['GetAcast/%s/%s' % (channel, display_id)]
-
+        cast_data = self._download_json(
+            'https://embed.acast.com/api/acasts/%s/%s' % (channel, display_id), display_id)

         return {
             'id': compat_str(cast_data['id']),
             'display_id': display_id,
         return {
             'id': compat_str(cast_data['id']),
             'display_id': display_id,

-            'url': cast_data['blings'][0]['audio'],
+            'url': [b['audio'] for b in cast_data['blings'] if b['type'] == 'BlingAudio'][0],

             'title': cast_data['name'],
             'description': cast_data.get('description'),
             'thumbnail': cast_data.get('image'),
             'title': cast_data['name'],
             'description': cast_data.get('description'),
             'thumbnail': cast_data.get('image'),

-            'timestamp': int_or_none(cast_data.get('publishingDate')),
+            'timestamp': parse_iso8601(cast_data.get('publishingDate')),

             'duration': int_or_none(cast_data.get('duration')),
         }
 
             'duration': int_or_none(cast_data.get('duration')),
         }
 


@@ -58,15 +73,26 @@ class ACastChannelIE(InfoExtractor):
         'playlist_mincount': 20,
     }
     _API_BASE_URL = 'https://www.acast.com/api/'
         'playlist_mincount': 20,
     }
     _API_BASE_URL = 'https://www.acast.com/api/'

+    _PAGE_SIZE = 10

 
     @classmethod
     def suitable(cls, url):
         return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)
 
 
     @classmethod
     def suitable(cls, url):
         return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)
 

-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        channel_data = self._download_json(self._API_BASE_URL + 'channels/%s' % display_id, display_id)
-        casts = self._download_json(self._API_BASE_URL + 'channels/%s/acasts' % display_id, display_id)
-        entries = [self.url_result('https://www.acast.com/%s/%s' % (display_id, cast['url']), 'ACast') for cast in casts]
+    def _fetch_page(self, channel_slug, page):
+        casts = self._download_json(
+            self._API_BASE_URL + 'channels/%s/acasts?page=%s' % (channel_slug, page),
+            channel_slug, note='Download page %d of channel data' % page)
+        for cast in casts:
+            yield self.url_result(
+                'https://www.acast.com/%s/%s' % (channel_slug, cast['url']),
+                'ACast', cast['id'])

 
 

-        return self.playlist_result(entries, compat_str(channel_data['id']), channel_data['name'], channel_data.get('description'))
+    def _real_extract(self, url):
+        channel_slug = self._match_id(url)
+        channel_data = self._download_json(
+            self._API_BASE_URL + 'channels/%s' % channel_slug, channel_slug)
+        entries = OnDemandPagedList(functools.partial(
+            self._fetch_page, channel_slug), self._PAGE_SIZE)
+        return self.playlist_result(entries, compat_str(
+            channel_data['id']), channel_data['name'], channel_data.get('description'))