]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ted.py
debian/patches: Remove patch to disambiguate between ffmpeg and avconv.
[youtubedl] / youtube_dl / extractor / ted.py
1 from __future__ import unicode_literals
2
3 import json
4 import re
5
6 from .common import InfoExtractor
7
8 from ..compat import compat_str
9 from ..utils import (
10 float_or_none,
11 int_or_none,
12 try_get,
13 url_or_none,
14 )
15
16
17 class TEDIE(InfoExtractor):
18 IE_NAME = 'ted'
19 _VALID_URL = r'''(?x)
20 (?P<proto>https?://)
21 (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
22 (
23 (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
24 |
25 ((?P<type_talk>talks)) # We have a simple talk
26 |
27 (?P<type_watch>watch)/[^/]+/[^/]+
28 )
29 (/lang/(.*?))? # The url may contain the language
30 /(?P<name>[\w-]+) # Here goes the name and then ".html"
31 .*)$
32 '''
33 _TESTS = [{
34 'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
35 'md5': 'b0ce2b05ca215042124fbc9e3886493a',
36 'info_dict': {
37 'id': '102',
38 'ext': 'mp4',
39 'title': 'The illusion of consciousness',
40 'description': ('Philosopher Dan Dennett makes a compelling '
41 'argument that not only don\'t we understand our own '
42 'consciousness, but that half the time our brains are '
43 'actively fooling us.'),
44 'uploader': 'Dan Dennett',
45 'width': 853,
46 'duration': 1308,
47 'view_count': int,
48 'comment_count': int,
49 'tags': list,
50 },
51 'params': {
52 'skip_download': True,
53 },
54 }, {
55 # missing HTTP bitrates
56 'url': 'https://www.ted.com/talks/vishal_sikka_the_beauty_and_power_of_algorithms',
57 'info_dict': {
58 'id': '6069',
59 'ext': 'mp4',
60 'title': 'The beauty and power of algorithms',
61 'thumbnail': r're:^https?://.+\.jpg',
62 'description': 'md5:734e352710fb00d840ab87ae31aaf688',
63 'uploader': 'Vishal Sikka',
64 },
65 'params': {
66 'skip_download': True,
67 },
68 }, {
69 'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
70 'md5': 'e6b9617c01a7970ceac8bb2c92c346c0',
71 'info_dict': {
72 'id': '1972',
73 'ext': 'mp4',
74 'title': 'Be passionate. Be courageous. Be your best.',
75 'uploader': 'Gabby Giffords and Mark Kelly',
76 'description': 'md5:5174aed4d0f16021b704120360f72b92',
77 'duration': 1128,
78 },
79 'params': {
80 'skip_download': True,
81 },
82 }, {
83 'url': 'http://www.ted.com/playlists/who_are_the_hackers',
84 'info_dict': {
85 'id': '10',
86 'title': 'Who are the hackers?',
87 },
88 'playlist_mincount': 6,
89 }, {
90 # contains a youtube video
91 'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
92 'add_ie': ['Youtube'],
93 'info_dict': {
94 'id': '_ZG8HBuDjgc',
95 'ext': 'webm',
96 'title': 'Douglas Adams: Parrots the Universe and Everything',
97 'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
98 'uploader': 'University of California Television (UCTV)',
99 'uploader_id': 'UCtelevision',
100 'upload_date': '20080522',
101 },
102 'params': {
103 'skip_download': True,
104 },
105 }, {
106 # no nativeDownloads
107 'url': 'https://www.ted.com/talks/tom_thum_the_orchestra_in_my_mouth',
108 'info_dict': {
109 'id': '1792',
110 'ext': 'mp4',
111 'title': 'The orchestra in my mouth',
112 'description': 'md5:5d1d78650e2f8dfcbb8ebee2951ac29a',
113 'uploader': 'Tom Thum',
114 'view_count': int,
115 'comment_count': int,
116 'tags': list,
117 },
118 'params': {
119 'skip_download': True,
120 },
121 }]
122
123 _NATIVE_FORMATS = {
124 'low': {'width': 320, 'height': 180},
125 'medium': {'width': 512, 'height': 288},
126 'high': {'width': 854, 'height': 480},
127 }
128
129 def _extract_info(self, webpage):
130 info_json = self._search_regex(
131 r'(?s)q\(\s*"\w+.init"\s*,\s*({.+})\)\s*</script>',
132 webpage, 'info json')
133 return json.loads(info_json)
134
135 def _real_extract(self, url):
136 m = re.match(self._VALID_URL, url, re.VERBOSE)
137 if m.group('type').startswith('embed'):
138 desktop_url = m.group('proto') + 'www' + m.group('urlmain')
139 return self.url_result(desktop_url, 'TED')
140 name = m.group('name')
141 if m.group('type_talk'):
142 return self._talk_info(url, name)
143 elif m.group('type_watch'):
144 return self._watch_info(url, name)
145 else:
146 return self._playlist_videos_info(url, name)
147
148 def _playlist_videos_info(self, url, name):
149 '''Returns the videos of the playlist'''
150
151 webpage = self._download_webpage(url, name,
152 'Downloading playlist webpage')
153 info = self._extract_info(webpage)
154
155 playlist_info = try_get(
156 info, lambda x: x['__INITIAL_DATA__']['playlist'],
157 dict) or info['playlist']
158
159 playlist_entries = [
160 self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
161 for talk in try_get(
162 info, lambda x: x['__INITIAL_DATA__']['talks'],
163 dict) or info['talks']
164 ]
165 return self.playlist_result(
166 playlist_entries,
167 playlist_id=compat_str(playlist_info['id']),
168 playlist_title=playlist_info['title'])
169
170 def _talk_info(self, url, video_name):
171 webpage = self._download_webpage(url, video_name)
172
173 info = self._extract_info(webpage)
174
175 data = try_get(info, lambda x: x['__INITIAL_DATA__'], dict) or info
176 talk_info = data['talks'][0]
177
178 title = talk_info['title'].strip()
179
180 native_downloads = try_get(
181 talk_info,
182 (lambda x: x['downloads']['nativeDownloads'],
183 lambda x: x['nativeDownloads']),
184 dict) or {}
185
186 formats = [{
187 'url': format_url,
188 'format_id': format_id,
189 'format': format_id,
190 } for (format_id, format_url) in native_downloads.items() if format_url is not None]
191 if formats:
192 for f in formats:
193 finfo = self._NATIVE_FORMATS.get(f['format_id'])
194 if finfo:
195 f.update(finfo)
196
197 player_talk = talk_info['player_talks'][0]
198
199 external = player_talk.get('external')
200 if isinstance(external, dict):
201 service = external.get('service')
202 if isinstance(service, compat_str):
203 ext_url = None
204 if service.lower() == 'youtube':
205 ext_url = external.get('code')
206
207 return self.url_result(ext_url or external['uri'])
208
209 resources_ = player_talk.get('resources') or talk_info.get('resources')
210
211 http_url = None
212 for format_id, resources in resources_.items():
213 if format_id == 'h264':
214 for resource in resources:
215 h264_url = resource.get('file')
216 if not h264_url:
217 continue
218 bitrate = int_or_none(resource.get('bitrate'))
219 formats.append({
220 'url': h264_url,
221 'format_id': '%s-%sk' % (format_id, bitrate),
222 'tbr': bitrate,
223 })
224 if re.search(r'\d+k', h264_url):
225 http_url = h264_url
226 elif format_id == 'rtmp':
227 streamer = talk_info.get('streamer')
228 if not streamer:
229 continue
230 for resource in resources:
231 formats.append({
232 'format_id': '%s-%s' % (format_id, resource.get('name')),
233 'url': streamer,
234 'play_path': resource['file'],
235 'ext': 'flv',
236 'width': int_or_none(resource.get('width')),
237 'height': int_or_none(resource.get('height')),
238 'tbr': int_or_none(resource.get('bitrate')),
239 })
240 elif format_id == 'hls':
241 if not isinstance(resources, dict):
242 continue
243 stream_url = url_or_none(resources.get('stream'))
244 if not stream_url:
245 continue
246 formats.extend(self._extract_m3u8_formats(
247 stream_url, video_name, 'mp4', m3u8_id=format_id,
248 fatal=False))
249
250 m3u8_formats = list(filter(
251 lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none',
252 formats))
253 if http_url:
254 for m3u8_format in m3u8_formats:
255 bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
256 if not bitrate:
257 continue
258 bitrate_url = re.sub(r'\d+k', bitrate, http_url)
259 if not self._is_valid_url(
260 bitrate_url, video_name, '%s bitrate' % bitrate):
261 continue
262 f = m3u8_format.copy()
263 f.update({
264 'url': bitrate_url,
265 'format_id': m3u8_format['format_id'].replace('hls', 'http'),
266 'protocol': 'http',
267 })
268 formats.append(f)
269
270 audio_download = talk_info.get('audioDownload')
271 if audio_download:
272 formats.append({
273 'url': audio_download,
274 'format_id': 'audio',
275 'vcodec': 'none',
276 })
277
278 self._sort_formats(formats)
279
280 video_id = compat_str(talk_info['id'])
281
282 return {
283 'id': video_id,
284 'title': title,
285 'uploader': player_talk.get('speaker') or talk_info.get('speaker'),
286 'thumbnail': player_talk.get('thumb') or talk_info.get('thumb'),
287 'description': self._og_search_description(webpage),
288 'subtitles': self._get_subtitles(video_id, talk_info),
289 'formats': formats,
290 'duration': float_or_none(talk_info.get('duration')),
291 'view_count': int_or_none(data.get('viewed_count')),
292 'comment_count': int_or_none(
293 try_get(data, lambda x: x['comments']['count'])),
294 'tags': try_get(talk_info, lambda x: x['tags'], list),
295 }
296
297 def _get_subtitles(self, video_id, talk_info):
298 sub_lang_list = {}
299 for language in try_get(
300 talk_info,
301 (lambda x: x['downloads']['languages'],
302 lambda x: x['languages']), list):
303 lang_code = language.get('languageCode') or language.get('ianaCode')
304 if not lang_code:
305 continue
306 sub_lang_list[lang_code] = [
307 {
308 'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, lang_code, ext),
309 'ext': ext,
310 }
311 for ext in ['ted', 'srt']
312 ]
313 return sub_lang_list
314
315 def _watch_info(self, url, name):
316 webpage = self._download_webpage(url, name)
317
318 config_json = self._html_search_regex(
319 r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
320 webpage, 'config', default=None)
321 if not config_json:
322 embed_url = self._search_regex(
323 r"<iframe[^>]+class='pages-video-embed__video__object'[^>]+src='([^']+)'", webpage, 'embed url')
324 return self.url_result(self._proto_relative_url(embed_url))
325 config = json.loads(config_json)['config']
326 video_url = config['video']['url']
327 thumbnail = config.get('image', {}).get('url')
328
329 title = self._html_search_regex(
330 r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
331 description = self._html_search_regex(
332 [
333 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
334 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
335 ],
336 webpage, 'description', fatal=False)
337
338 return {
339 'id': name,
340 'url': video_url,
341 'title': title,
342 'thumbnail': thumbnail,
343 'description': description,
344 }