]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ted.py
Prepare new release.
[youtubedl] / youtube_dl / extractor / ted.py
1 from __future__ import unicode_literals
2
3 import json
4 import re
5
6 from .common import InfoExtractor
7
8 from ..compat import compat_str
9 from ..utils import int_or_none
10
11
12 class TEDIE(InfoExtractor):
13 IE_NAME = 'ted'
14 _VALID_URL = r'''(?x)
15 (?P<proto>https?://)
16 (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
17 (
18 (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
19 |
20 ((?P<type_talk>talks)) # We have a simple talk
21 |
22 (?P<type_watch>watch)/[^/]+/[^/]+
23 )
24 (/lang/(.*?))? # The url may contain the language
25 /(?P<name>[\w-]+) # Here goes the name and then ".html"
26 .*)$
27 '''
28 _TESTS = [{
29 'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
30 'md5': '0de43ac406aa3e4ea74b66c9c7789b13',
31 'info_dict': {
32 'id': '102',
33 'ext': 'mp4',
34 'title': 'The illusion of consciousness',
35 'description': ('Philosopher Dan Dennett makes a compelling '
36 'argument that not only don\'t we understand our own '
37 'consciousness, but that half the time our brains are '
38 'actively fooling us.'),
39 'uploader': 'Dan Dennett',
40 'width': 853,
41 'duration': 1308,
42 }
43 }, {
44 'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
45 'md5': 'b899ac15e345fb39534d913f7606082b',
46 'info_dict': {
47 'id': 'tSVI8ta_P4w',
48 'ext': 'mp4',
49 'title': 'Vishal Sikka: The beauty and power of algorithms',
50 'thumbnail': r're:^https?://.+\.jpg',
51 'description': 'md5:6261fdfe3e02f4f579cbbfc00aff73f4',
52 'upload_date': '20140122',
53 'uploader_id': 'TEDInstitute',
54 'uploader': 'TED Institute',
55 },
56 'add_ie': ['Youtube'],
57 }, {
58 'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
59 'md5': '71b3ab2f4233012dce09d515c9c39ce2',
60 'info_dict': {
61 'id': '1972',
62 'ext': 'mp4',
63 'title': 'Be passionate. Be courageous. Be your best.',
64 'uploader': 'Gabby Giffords and Mark Kelly',
65 'description': 'md5:5174aed4d0f16021b704120360f72b92',
66 'duration': 1128,
67 },
68 }, {
69 'url': 'http://www.ted.com/playlists/who_are_the_hackers',
70 'info_dict': {
71 'id': '10',
72 'title': 'Who are the hackers?',
73 },
74 'playlist_mincount': 6,
75 }, {
76 # contains a youtube video
77 'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
78 'add_ie': ['Youtube'],
79 'info_dict': {
80 'id': '_ZG8HBuDjgc',
81 'ext': 'webm',
82 'title': 'Douglas Adams: Parrots the Universe and Everything',
83 'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
84 'uploader': 'University of California Television (UCTV)',
85 'uploader_id': 'UCtelevision',
86 'upload_date': '20080522',
87 },
88 'params': {
89 'skip_download': True,
90 },
91 }, {
92 # YouTube video
93 'url': 'http://www.ted.com/talks/jeffrey_kluger_the_sibling_bond',
94 'add_ie': ['Youtube'],
95 'info_dict': {
96 'id': 'aFBIPO-P7LM',
97 'ext': 'mp4',
98 'title': 'The hidden power of siblings: Jeff Kluger at TEDxAsheville',
99 'description': 'md5:3d7a4f50d95ca5dd67104e2a20f43fe1',
100 'uploader': 'TEDx Talks',
101 'uploader_id': 'TEDxTalks',
102 'upload_date': '20111216',
103 },
104 'params': {
105 'skip_download': True,
106 },
107 }]
108
109 _NATIVE_FORMATS = {
110 'low': {'width': 320, 'height': 180},
111 'medium': {'width': 512, 'height': 288},
112 'high': {'width': 854, 'height': 480},
113 }
114
115 def _extract_info(self, webpage):
116 info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
117 webpage, 'info json')
118 return json.loads(info_json)
119
120 def _real_extract(self, url):
121 m = re.match(self._VALID_URL, url, re.VERBOSE)
122 if m.group('type').startswith('embed'):
123 desktop_url = m.group('proto') + 'www' + m.group('urlmain')
124 return self.url_result(desktop_url, 'TED')
125 name = m.group('name')
126 if m.group('type_talk'):
127 return self._talk_info(url, name)
128 elif m.group('type_watch'):
129 return self._watch_info(url, name)
130 else:
131 return self._playlist_videos_info(url, name)
132
133 def _playlist_videos_info(self, url, name):
134 '''Returns the videos of the playlist'''
135
136 webpage = self._download_webpage(url, name,
137 'Downloading playlist webpage')
138 info = self._extract_info(webpage)
139 playlist_info = info['playlist']
140
141 playlist_entries = [
142 self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
143 for talk in info['talks']
144 ]
145 return self.playlist_result(
146 playlist_entries,
147 playlist_id=compat_str(playlist_info['id']),
148 playlist_title=playlist_info['title'])
149
150 def _talk_info(self, url, video_name):
151 webpage = self._download_webpage(url, video_name)
152 self.report_extraction(video_name)
153
154 talk_info = self._extract_info(webpage)['talks'][0]
155
156 external = talk_info.get('external')
157 if external:
158 service = external['service']
159 self.to_screen('Found video from %s' % service)
160 ext_url = None
161 if service.lower() == 'youtube':
162 ext_url = external.get('code')
163 return {
164 '_type': 'url',
165 'url': ext_url or external['uri'],
166 }
167
168 formats = [{
169 'url': format_url,
170 'format_id': format_id,
171 'format': format_id,
172 } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
173 if formats:
174 for f in formats:
175 finfo = self._NATIVE_FORMATS.get(f['format_id'])
176 if finfo:
177 f.update(finfo)
178
179 http_url = None
180 for format_id, resources in talk_info['resources'].items():
181 if format_id == 'h264':
182 for resource in resources:
183 h264_url = resource.get('file')
184 if not h264_url:
185 continue
186 bitrate = int_or_none(resource.get('bitrate'))
187 formats.append({
188 'url': h264_url,
189 'format_id': '%s-%sk' % (format_id, bitrate),
190 'tbr': bitrate,
191 })
192 if re.search(r'\d+k', h264_url):
193 http_url = h264_url
194 elif format_id == 'rtmp':
195 streamer = talk_info.get('streamer')
196 if not streamer:
197 continue
198 for resource in resources:
199 formats.append({
200 'format_id': '%s-%s' % (format_id, resource.get('name')),
201 'url': streamer,
202 'play_path': resource['file'],
203 'ext': 'flv',
204 'width': int_or_none(resource.get('width')),
205 'height': int_or_none(resource.get('height')),
206 'tbr': int_or_none(resource.get('bitrate')),
207 })
208 elif format_id == 'hls':
209 formats.extend(self._extract_m3u8_formats(
210 resources.get('stream'), video_name, 'mp4', m3u8_id=format_id, fatal=False))
211
212 m3u8_formats = list(filter(
213 lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
214 formats))
215 if http_url:
216 for m3u8_format in m3u8_formats:
217 bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
218 if not bitrate:
219 continue
220 f = m3u8_format.copy()
221 f.update({
222 'url': re.sub(r'\d+k', bitrate, http_url),
223 'format_id': m3u8_format['format_id'].replace('hls', 'http'),
224 'protocol': 'http',
225 })
226 formats.append(f)
227
228 audio_download = talk_info.get('audioDownload')
229 if audio_download:
230 formats.append({
231 'url': audio_download,
232 'format_id': 'audio',
233 'vcodec': 'none',
234 })
235
236 self._sort_formats(formats)
237
238 video_id = compat_str(talk_info['id'])
239
240 thumbnail = talk_info['thumb']
241 if not thumbnail.startswith('http'):
242 thumbnail = 'http://' + thumbnail
243 return {
244 'id': video_id,
245 'title': talk_info['title'].strip(),
246 'uploader': talk_info['speaker'],
247 'thumbnail': thumbnail,
248 'description': self._og_search_description(webpage),
249 'subtitles': self._get_subtitles(video_id, talk_info),
250 'formats': formats,
251 'duration': talk_info.get('duration'),
252 }
253
254 def _get_subtitles(self, video_id, talk_info):
255 languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
256 if languages:
257 sub_lang_list = {}
258 for l in languages:
259 sub_lang_list[l] = [
260 {
261 'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, l, ext),
262 'ext': ext,
263 }
264 for ext in ['ted', 'srt']
265 ]
266 return sub_lang_list
267 else:
268 return {}
269
270 def _watch_info(self, url, name):
271 webpage = self._download_webpage(url, name)
272
273 config_json = self._html_search_regex(
274 r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
275 webpage, 'config', default=None)
276 if not config_json:
277 embed_url = self._search_regex(
278 r"<iframe[^>]+class='pages-video-embed__video__object'[^>]+src='([^']+)'", webpage, 'embed url')
279 return self.url_result(self._proto_relative_url(embed_url))
280 config = json.loads(config_json)['config']
281 video_url = config['video']['url']
282 thumbnail = config.get('image', {}).get('url')
283
284 title = self._html_search_regex(
285 r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
286 description = self._html_search_regex(
287 [
288 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
289 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
290 ],
291 webpage, 'description', fatal=False)
292
293 return {
294 'id': name,
295 'url': video_url,
296 'title': title,
297 'thumbnail': thumbnail,
298 'description': description,
299 }