]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ted.py
cd4af96fdb02b2f4ea392fe19482371b6a02acad
[youtubedl] / youtube_dl / extractor / ted.py
1 from __future__ import unicode_literals
2
3 import json
4 import re
5
6 from .subtitles import SubtitlesInfoExtractor
7
8 from ..utils import (
9 compat_str,
10 )
11
12
13 class TEDIE(SubtitlesInfoExtractor):
14 _VALID_URL = r'''(?x)
15 (?P<proto>https?://)
16 (?P<type>www|embed)(?P<urlmain>\.ted\.com/
17 (
18 (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
19 |
20 ((?P<type_talk>talks)) # We have a simple talk
21 |
22 (?P<type_watch>watch)/[^/]+/[^/]+
23 )
24 (/lang/(.*?))? # The url may contain the language
25 /(?P<name>[\w-]+) # Here goes the name and then ".html"
26 .*)$
27 '''
28 _TESTS = [{
29 'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
30 'md5': 'fc94ac279feebbce69f21c0c6ee82810',
31 'info_dict': {
32 'id': '102',
33 'ext': 'mp4',
34 'title': 'The illusion of consciousness',
35 'description': ('Philosopher Dan Dennett makes a compelling '
36 'argument that not only don\'t we understand our own '
37 'consciousness, but that half the time our brains are '
38 'actively fooling us.'),
39 'uploader': 'Dan Dennett',
40 'width': 854,
41 }
42 }, {
43 'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
44 'md5': '226f4fb9c62380d11b7995efa4c87994',
45 'info_dict': {
46 'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
47 'ext': 'mp4',
48 'title': 'Vishal Sikka: The beauty and power of algorithms',
49 'thumbnail': 're:^https?://.+\.jpg',
50 'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
51 }
52 }, {
53 'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
54 'info_dict': {
55 'id': '1972',
56 'ext': 'mp4',
57 'title': 'Be passionate. Be courageous. Be your best.',
58 'uploader': 'Gabby Giffords and Mark Kelly',
59 'description': 'md5:5174aed4d0f16021b704120360f72b92',
60 },
61 }, {
62 'url': 'http://www.ted.com/playlists/who_are_the_hackers',
63 'info_dict': {
64 'id': '10',
65 'title': 'Who are the hackers?',
66 },
67 'playlist_mincount': 6,
68 }, {
69 # contains a youtube video
70 'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
71 'add_ie': ['Youtube'],
72 'info_dict': {
73 'id': '_ZG8HBuDjgc',
74 'ext': 'mp4',
75 'title': 'Douglas Adams: Parrots the Universe and Everything',
76 'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
77 'uploader': 'University of California Television (UCTV)',
78 'uploader_id': 'UCtelevision',
79 'upload_date': '20080522',
80 },
81 'params': {
82 'skip_download': True,
83 },
84 }]
85
86 _NATIVE_FORMATS = {
87 'low': {'preference': 1, 'width': 320, 'height': 180},
88 'medium': {'preference': 2, 'width': 512, 'height': 288},
89 'high': {'preference': 3, 'width': 854, 'height': 480},
90 }
91
92 def _extract_info(self, webpage):
93 info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
94 webpage, 'info json')
95 return json.loads(info_json)
96
97 def _real_extract(self, url):
98 m = re.match(self._VALID_URL, url, re.VERBOSE)
99 if m.group('type') == 'embed':
100 desktop_url = m.group('proto') + 'www' + m.group('urlmain')
101 return self.url_result(desktop_url, 'TED')
102 name = m.group('name')
103 if m.group('type_talk'):
104 return self._talk_info(url, name)
105 elif m.group('type_watch'):
106 return self._watch_info(url, name)
107 else:
108 return self._playlist_videos_info(url, name)
109
110 def _playlist_videos_info(self, url, name):
111 '''Returns the videos of the playlist'''
112
113 webpage = self._download_webpage(url, name,
114 'Downloading playlist webpage')
115 info = self._extract_info(webpage)
116 playlist_info = info['playlist']
117
118 playlist_entries = [
119 self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
120 for talk in info['talks']
121 ]
122 return self.playlist_result(
123 playlist_entries,
124 playlist_id=compat_str(playlist_info['id']),
125 playlist_title=playlist_info['title'])
126
127 def _talk_info(self, url, video_name):
128 webpage = self._download_webpage(url, video_name)
129 self.report_extraction(video_name)
130
131 talk_info = self._extract_info(webpage)['talks'][0]
132
133 if talk_info.get('external') is not None:
134 self.to_screen('Found video from %s' % talk_info['external']['service'])
135 return {
136 '_type': 'url',
137 'url': talk_info['external']['uri'],
138 }
139
140 formats = [{
141 'url': format_url,
142 'format_id': format_id,
143 'format': format_id,
144 } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
145 if formats:
146 for f in formats:
147 finfo = self._NATIVE_FORMATS.get(f['format_id'])
148 if finfo:
149 f.update(finfo)
150 else:
151 # Use rtmp downloads
152 formats = [{
153 'format_id': f['name'],
154 'url': talk_info['streamer'],
155 'play_path': f['file'],
156 'ext': 'flv',
157 'width': f['width'],
158 'height': f['height'],
159 'tbr': f['bitrate'],
160 } for f in talk_info['resources']['rtmp']]
161 self._sort_formats(formats)
162
163 video_id = compat_str(talk_info['id'])
164 # subtitles
165 video_subtitles = self.extract_subtitles(video_id, talk_info)
166 if self._downloader.params.get('listsubtitles', False):
167 self._list_available_subtitles(video_id, talk_info)
168 return
169
170 thumbnail = talk_info['thumb']
171 if not thumbnail.startswith('http'):
172 thumbnail = 'http://' + thumbnail
173 return {
174 'id': video_id,
175 'title': talk_info['title'].strip(),
176 'uploader': talk_info['speaker'],
177 'thumbnail': thumbnail,
178 'description': self._og_search_description(webpage),
179 'subtitles': video_subtitles,
180 'formats': formats,
181 }
182
183 def _get_available_subtitles(self, video_id, talk_info):
184 languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
185 if languages:
186 sub_lang_list = {}
187 for l in languages:
188 url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
189 sub_lang_list[l] = url
190 return sub_lang_list
191 else:
192 self._downloader.report_warning('video doesn\'t have subtitles')
193 return {}
194
195 def _watch_info(self, url, name):
196 webpage = self._download_webpage(url, name)
197
198 config_json = self._html_search_regex(
199 r"data-config='([^']+)", webpage, 'config')
200 config = json.loads(config_json)
201 video_url = config['video']['url']
202 thumbnail = config.get('image', {}).get('url')
203
204 title = self._html_search_regex(
205 r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
206 description = self._html_search_regex(
207 [
208 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
209 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
210 ],
211 webpage, 'description', fatal=False)
212
213 return {
214 'id': name,
215 'url': video_url,
216 'title': title,
217 'thumbnail': thumbnail,
218 'description': description,
219 }