X-Git-Url: https://git.rapsys.eu/youtubedl/blobdiff_plain/22bc55bffeb45b7d2f3056ae863eb3228e6507e8..39393b81acfaf4045fb7f20454a0226f0dc9142e:/youtube_dl/extractor/ted.py diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py index 212ac80..63e2455 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dl/extractor/ted.py @@ -5,8 +5,12 @@ import re from .common import InfoExtractor -from ..compat import compat_str +from ..compat import ( + compat_str, + compat_urlparse +) from ..utils import ( + extract_attributes, float_or_none, int_or_none, try_get, @@ -20,7 +24,7 @@ class TEDIE(InfoExtractor): (?Phttps?://) (?Pwww|embed(?:-ssl)?)(?P\.ted\.com/ ( - (?Pplaylists(?:/\d+)?) # We have a playlist + (?Pplaylists(?:/(?P\d+))?) # We have a playlist | ((?Ptalks)) # We have a simple talk | @@ -84,6 +88,7 @@ class TEDIE(InfoExtractor): 'info_dict': { 'id': '10', 'title': 'Who are the hackers?', + 'description': 'md5:49a0dbe8fb76d81a0e64b4a80af7f15a' }, 'playlist_mincount': 6, }, { @@ -128,7 +133,7 @@ class TEDIE(InfoExtractor): def _extract_info(self, webpage): info_json = self._search_regex( - r'(?s)q\(\s*"\w+.init"\s*,\s*({.+})\)\s*', + r'(?s)q\(\s*"\w+.init"\s*,\s*({.+?})\)\s*', webpage, 'info json') return json.loads(info_json) @@ -150,22 +155,22 @@ class TEDIE(InfoExtractor): webpage = self._download_webpage(url, name, 'Downloading playlist webpage') - info = self._extract_info(webpage) - playlist_info = try_get( - info, lambda x: x['__INITIAL_DATA__']['playlist'], - dict) or info['playlist'] + playlist_entries = [] + for entry in re.findall(r'(?s)<[^>]+data-ga-context=["\']playlist["\'][^>]*>', webpage): + attrs = extract_attributes(entry) + entry_url = compat_urlparse.urljoin(url, attrs['href']) + playlist_entries.append(self.url_result(entry_url, self.ie_key())) + + final_url = self._og_search_url(webpage, fatal=False) + playlist_id = ( + re.match(self._VALID_URL, final_url).group('playlist_id') + if final_url else None) - playlist_entries = [ - self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key()) - for talk in try_get( - info, lambda x: x['__INITIAL_DATA__']['talks'], - dict) or info['talks'] - ] return self.playlist_result( - playlist_entries, - playlist_id=compat_str(playlist_info['id']), - playlist_title=playlist_info['title']) + playlist_entries, playlist_id=playlist_id, + playlist_title=self._og_search_title(webpage, fatal=False), + playlist_description=self._og_search_description(webpage)) def _talk_info(self, url, video_name): webpage = self._download_webpage(url, video_name) @@ -177,20 +182,29 @@ class TEDIE(InfoExtractor): title = talk_info['title'].strip() - native_downloads = try_get( - talk_info, - (lambda x: x['downloads']['nativeDownloads'], - lambda x: x['nativeDownloads']), - dict) or {} + downloads = talk_info.get('downloads') or {} + native_downloads = downloads.get('nativeDownloads') or talk_info.get('nativeDownloads') or {} formats = [{ 'url': format_url, 'format_id': format_id, - 'format': format_id, } for (format_id, format_url) in native_downloads.items() if format_url is not None] + + subtitled_downloads = downloads.get('subtitledDownloads') or {} + for lang, subtitled_download in subtitled_downloads.items(): + for q in self._NATIVE_FORMATS: + q_url = subtitled_download.get(q) + if not q_url: + continue + formats.append({ + 'url': q_url, + 'format_id': '%s-%s' % (q, lang), + 'language': lang, + }) + if formats: for f in formats: - finfo = self._NATIVE_FORMATS.get(f['format_id']) + finfo = self._NATIVE_FORMATS.get(f['format_id'].split('-')[0]) if finfo: f.update(finfo) @@ -203,51 +217,52 @@ class TEDIE(InfoExtractor): ext_url = None if service.lower() == 'youtube': ext_url = external.get('code') - return { - '_type': 'url', - 'url': ext_url or external['uri'], - } + + return self.url_result(ext_url or external['uri']) resources_ = player_talk.get('resources') or talk_info.get('resources') http_url = None for format_id, resources in resources_.items(): - if not isinstance(resources, dict): - continue - if format_id == 'h264': - for resource in resources: - h264_url = resource.get('file') - if not h264_url: - continue - bitrate = int_or_none(resource.get('bitrate')) - formats.append({ - 'url': h264_url, - 'format_id': '%s-%sk' % (format_id, bitrate), - 'tbr': bitrate, - }) - if re.search(r'\d+k', h264_url): - http_url = h264_url - elif format_id == 'rtmp': - streamer = talk_info.get('streamer') - if not streamer: + if format_id == 'hls': + if not isinstance(resources, dict): continue - for resource in resources: - formats.append({ - 'format_id': '%s-%s' % (format_id, resource.get('name')), - 'url': streamer, - 'play_path': resource['file'], - 'ext': 'flv', - 'width': int_or_none(resource.get('width')), - 'height': int_or_none(resource.get('height')), - 'tbr': int_or_none(resource.get('bitrate')), - }) - elif format_id == 'hls': stream_url = url_or_none(resources.get('stream')) if not stream_url: continue formats.extend(self._extract_m3u8_formats( stream_url, video_name, 'mp4', m3u8_id=format_id, fatal=False)) + else: + if not isinstance(resources, list): + continue + if format_id == 'h264': + for resource in resources: + h264_url = resource.get('file') + if not h264_url: + continue + bitrate = int_or_none(resource.get('bitrate')) + formats.append({ + 'url': h264_url, + 'format_id': '%s-%sk' % (format_id, bitrate), + 'tbr': bitrate, + }) + if re.search(r'\d+k', h264_url): + http_url = h264_url + elif format_id == 'rtmp': + streamer = talk_info.get('streamer') + if not streamer: + continue + for resource in resources: + formats.append({ + 'format_id': '%s-%s' % (format_id, resource.get('name')), + 'url': streamer, + 'play_path': resource['file'], + 'ext': 'flv', + 'width': int_or_none(resource.get('width')), + 'height': int_or_none(resource.get('height')), + 'tbr': int_or_none(resource.get('bitrate')), + }) m3u8_formats = list(filter( lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none', @@ -267,6 +282,8 @@ class TEDIE(InfoExtractor): 'format_id': m3u8_format['format_id'].replace('hls', 'http'), 'protocol': 'http', }) + if f.get('acodec') == 'none': + del f['acodec'] formats.append(f) audio_download = talk_info.get('audioDownload')