]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ted.py
Update changelog.
[youtubedl] / youtube_dl / extractor / ted.py
1 from __future__ import unicode_literals
2
3 import json
4 import re
5
6 from .common import InfoExtractor
7
8 from ..compat import compat_str
9 from ..utils import (
10 float_or_none,
11 int_or_none,
12 try_get,
13 url_or_none,
14 )
15
16
17 class TEDIE(InfoExtractor):
18 IE_NAME = 'ted'
19 _VALID_URL = r'''(?x)
20 (?P<proto>https?://)
21 (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
22 (
23 (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
24 |
25 ((?P<type_talk>talks)) # We have a simple talk
26 |
27 (?P<type_watch>watch)/[^/]+/[^/]+
28 )
29 (/lang/(.*?))? # The url may contain the language
30 /(?P<name>[\w-]+) # Here goes the name and then ".html"
31 .*)$
32 '''
33 _TESTS = [{
34 'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
35 'md5': 'b0ce2b05ca215042124fbc9e3886493a',
36 'info_dict': {
37 'id': '102',
38 'ext': 'mp4',
39 'title': 'The illusion of consciousness',
40 'description': ('Philosopher Dan Dennett makes a compelling '
41 'argument that not only don\'t we understand our own '
42 'consciousness, but that half the time our brains are '
43 'actively fooling us.'),
44 'uploader': 'Dan Dennett',
45 'width': 853,
46 'duration': 1308,
47 'view_count': int,
48 'comment_count': int,
49 'tags': list,
50 },
51 'params': {
52 'skip_download': True,
53 },
54 }, {
55 # missing HTTP bitrates
56 'url': 'https://www.ted.com/talks/vishal_sikka_the_beauty_and_power_of_algorithms',
57 'info_dict': {
58 'id': '6069',
59 'ext': 'mp4',
60 'title': 'The beauty and power of algorithms',
61 'thumbnail': r're:^https?://.+\.jpg',
62 'description': 'md5:734e352710fb00d840ab87ae31aaf688',
63 'uploader': 'Vishal Sikka',
64 },
65 'params': {
66 'skip_download': True,
67 },
68 }, {
69 'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
70 'md5': 'e6b9617c01a7970ceac8bb2c92c346c0',
71 'info_dict': {
72 'id': '1972',
73 'ext': 'mp4',
74 'title': 'Be passionate. Be courageous. Be your best.',
75 'uploader': 'Gabby Giffords and Mark Kelly',
76 'description': 'md5:5174aed4d0f16021b704120360f72b92',
77 'duration': 1128,
78 },
79 'params': {
80 'skip_download': True,
81 },
82 }, {
83 'url': 'http://www.ted.com/playlists/who_are_the_hackers',
84 'info_dict': {
85 'id': '10',
86 'title': 'Who are the hackers?',
87 },
88 'playlist_mincount': 6,
89 }, {
90 # contains a youtube video
91 'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
92 'add_ie': ['Youtube'],
93 'info_dict': {
94 'id': '_ZG8HBuDjgc',
95 'ext': 'webm',
96 'title': 'Douglas Adams: Parrots the Universe and Everything',
97 'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
98 'uploader': 'University of California Television (UCTV)',
99 'uploader_id': 'UCtelevision',
100 'upload_date': '20080522',
101 },
102 'params': {
103 'skip_download': True,
104 },
105 }, {
106 # no nativeDownloads
107 'url': 'https://www.ted.com/talks/tom_thum_the_orchestra_in_my_mouth',
108 'info_dict': {
109 'id': '1792',
110 'ext': 'mp4',
111 'title': 'The orchestra in my mouth',
112 'description': 'md5:5d1d78650e2f8dfcbb8ebee2951ac29a',
113 'uploader': 'Tom Thum',
114 'view_count': int,
115 'comment_count': int,
116 'tags': list,
117 },
118 'params': {
119 'skip_download': True,
120 },
121 }]
122
123 _NATIVE_FORMATS = {
124 'low': {'width': 320, 'height': 180},
125 'medium': {'width': 512, 'height': 288},
126 'high': {'width': 854, 'height': 480},
127 }
128
129 def _extract_info(self, webpage):
130 info_json = self._search_regex(
131 r'(?s)q\(\s*"\w+.init"\s*,\s*({.+})\)\s*</script>',
132 webpage, 'info json')
133 return json.loads(info_json)
134
135 def _real_extract(self, url):
136 m = re.match(self._VALID_URL, url, re.VERBOSE)
137 if m.group('type').startswith('embed'):
138 desktop_url = m.group('proto') + 'www' + m.group('urlmain')
139 return self.url_result(desktop_url, 'TED')
140 name = m.group('name')
141 if m.group('type_talk'):
142 return self._talk_info(url, name)
143 elif m.group('type_watch'):
144 return self._watch_info(url, name)
145 else:
146 return self._playlist_videos_info(url, name)
147
148 def _playlist_videos_info(self, url, name):
149 '''Returns the videos of the playlist'''
150
151 webpage = self._download_webpage(url, name,
152 'Downloading playlist webpage')
153 info = self._extract_info(webpage)
154
155 playlist_info = try_get(
156 info, lambda x: x['__INITIAL_DATA__']['playlist'],
157 dict) or info['playlist']
158
159 playlist_entries = [
160 self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
161 for talk in try_get(
162 info, lambda x: x['__INITIAL_DATA__']['talks'],
163 dict) or info['talks']
164 ]
165 return self.playlist_result(
166 playlist_entries,
167 playlist_id=compat_str(playlist_info['id']),
168 playlist_title=playlist_info['title'])
169
170 def _talk_info(self, url, video_name):
171 webpage = self._download_webpage(url, video_name)
172
173 info = self._extract_info(webpage)
174
175 data = try_get(info, lambda x: x['__INITIAL_DATA__'], dict) or info
176 talk_info = data['talks'][0]
177
178 title = talk_info['title'].strip()
179
180 native_downloads = try_get(
181 talk_info,
182 (lambda x: x['downloads']['nativeDownloads'],
183 lambda x: x['nativeDownloads']),
184 dict) or {}
185
186 formats = [{
187 'url': format_url,
188 'format_id': format_id,
189 'format': format_id,
190 } for (format_id, format_url) in native_downloads.items() if format_url is not None]
191 if formats:
192 for f in formats:
193 finfo = self._NATIVE_FORMATS.get(f['format_id'])
194 if finfo:
195 f.update(finfo)
196
197 player_talk = talk_info['player_talks'][0]
198
199 external = player_talk.get('external')
200 if isinstance(external, dict):
201 service = external.get('service')
202 if isinstance(service, compat_str):
203 ext_url = None
204 if service.lower() == 'youtube':
205 ext_url = external.get('code')
206 return {
207 '_type': 'url',
208 'url': ext_url or external['uri'],
209 }
210
211 resources_ = player_talk.get('resources') or talk_info.get('resources')
212
213 http_url = None
214 for format_id, resources in resources_.items():
215 if format_id == 'h264':
216 for resource in resources:
217 h264_url = resource.get('file')
218 if not h264_url:
219 continue
220 bitrate = int_or_none(resource.get('bitrate'))
221 formats.append({
222 'url': h264_url,
223 'format_id': '%s-%sk' % (format_id, bitrate),
224 'tbr': bitrate,
225 })
226 if re.search(r'\d+k', h264_url):
227 http_url = h264_url
228 elif format_id == 'rtmp':
229 streamer = talk_info.get('streamer')
230 if not streamer:
231 continue
232 for resource in resources:
233 formats.append({
234 'format_id': '%s-%s' % (format_id, resource.get('name')),
235 'url': streamer,
236 'play_path': resource['file'],
237 'ext': 'flv',
238 'width': int_or_none(resource.get('width')),
239 'height': int_or_none(resource.get('height')),
240 'tbr': int_or_none(resource.get('bitrate')),
241 })
242 elif format_id == 'hls':
243 if not isinstance(resources, dict):
244 continue
245 stream_url = url_or_none(resources.get('stream'))
246 if not stream_url:
247 continue
248 formats.extend(self._extract_m3u8_formats(
249 stream_url, video_name, 'mp4', m3u8_id=format_id,
250 fatal=False))
251
252 m3u8_formats = list(filter(
253 lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none',
254 formats))
255 if http_url:
256 for m3u8_format in m3u8_formats:
257 bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
258 if not bitrate:
259 continue
260 bitrate_url = re.sub(r'\d+k', bitrate, http_url)
261 if not self._is_valid_url(
262 bitrate_url, video_name, '%s bitrate' % bitrate):
263 continue
264 f = m3u8_format.copy()
265 f.update({
266 'url': bitrate_url,
267 'format_id': m3u8_format['format_id'].replace('hls', 'http'),
268 'protocol': 'http',
269 })
270 formats.append(f)
271
272 audio_download = talk_info.get('audioDownload')
273 if audio_download:
274 formats.append({
275 'url': audio_download,
276 'format_id': 'audio',
277 'vcodec': 'none',
278 })
279
280 self._sort_formats(formats)
281
282 video_id = compat_str(talk_info['id'])
283
284 return {
285 'id': video_id,
286 'title': title,
287 'uploader': player_talk.get('speaker') or talk_info.get('speaker'),
288 'thumbnail': player_talk.get('thumb') or talk_info.get('thumb'),
289 'description': self._og_search_description(webpage),
290 'subtitles': self._get_subtitles(video_id, talk_info),
291 'formats': formats,
292 'duration': float_or_none(talk_info.get('duration')),
293 'view_count': int_or_none(data.get('viewed_count')),
294 'comment_count': int_or_none(
295 try_get(data, lambda x: x['comments']['count'])),
296 'tags': try_get(talk_info, lambda x: x['tags'], list),
297 }
298
299 def _get_subtitles(self, video_id, talk_info):
300 sub_lang_list = {}
301 for language in try_get(
302 talk_info,
303 (lambda x: x['downloads']['languages'],
304 lambda x: x['languages']), list):
305 lang_code = language.get('languageCode') or language.get('ianaCode')
306 if not lang_code:
307 continue
308 sub_lang_list[lang_code] = [
309 {
310 'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, lang_code, ext),
311 'ext': ext,
312 }
313 for ext in ['ted', 'srt']
314 ]
315 return sub_lang_list
316
317 def _watch_info(self, url, name):
318 webpage = self._download_webpage(url, name)
319
320 config_json = self._html_search_regex(
321 r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
322 webpage, 'config', default=None)
323 if not config_json:
324 embed_url = self._search_regex(
325 r"<iframe[^>]+class='pages-video-embed__video__object'[^>]+src='([^']+)'", webpage, 'embed url')
326 return self.url_result(self._proto_relative_url(embed_url))
327 config = json.loads(config_json)['config']
328 video_url = config['video']['url']
329 thumbnail = config.get('image', {}).get('url')
330
331 title = self._html_search_regex(
332 r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
333 description = self._html_search_regex(
334 [
335 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
336 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
337 ],
338 webpage, 'description', fatal=False)
339
340 return {
341 'id': name,
342 'url': video_url,
343 'title': title,
344 'thumbnail': thumbnail,
345 'description': description,
346 }