]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ted.py
Prepare for upload.
[youtubedl] / youtube_dl / extractor / ted.py
1 from __future__ import unicode_literals
2
3 import json
4 import re
5
6 from .subtitles import SubtitlesInfoExtractor
7
8 from ..utils import (
9 compat_str,
10 )
11
12
13 class TEDIE(SubtitlesInfoExtractor):
14 _VALID_URL = r'''(?x)
15 (?P<proto>https?://)
16 (?P<type>www|embed)(?P<urlmain>\.ted\.com/
17 (
18 (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
19 |
20 ((?P<type_talk>talks)) # We have a simple talk
21 |
22 (?P<type_watch>watch)/[^/]+/[^/]+
23 )
24 (/lang/(.*?))? # The url may contain the language
25 /(?P<name>[\w-]+) # Here goes the name and then ".html"
26 .*)$
27 '''
28 _TESTS = [{
29 'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
30 'md5': 'fc94ac279feebbce69f21c0c6ee82810',
31 'info_dict': {
32 'id': '102',
33 'ext': 'mp4',
34 'title': 'The illusion of consciousness',
35 'description': ('Philosopher Dan Dennett makes a compelling '
36 'argument that not only don\'t we understand our own '
37 'consciousness, but that half the time our brains are '
38 'actively fooling us.'),
39 'uploader': 'Dan Dennett',
40 'width': 854,
41 'duration': 1308,
42 }
43 }, {
44 'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
45 'md5': '226f4fb9c62380d11b7995efa4c87994',
46 'info_dict': {
47 'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
48 'ext': 'mp4',
49 'title': 'Vishal Sikka: The beauty and power of algorithms',
50 'thumbnail': 're:^https?://.+\.jpg',
51 'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
52 }
53 }, {
54 'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
55 'info_dict': {
56 'id': '1972',
57 'ext': 'mp4',
58 'title': 'Be passionate. Be courageous. Be your best.',
59 'uploader': 'Gabby Giffords and Mark Kelly',
60 'description': 'md5:5174aed4d0f16021b704120360f72b92',
61 'duration': 1128,
62 },
63 }, {
64 'url': 'http://www.ted.com/playlists/who_are_the_hackers',
65 'info_dict': {
66 'id': '10',
67 'title': 'Who are the hackers?',
68 },
69 'playlist_mincount': 6,
70 }, {
71 # contains a youtube video
72 'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
73 'add_ie': ['Youtube'],
74 'info_dict': {
75 'id': '_ZG8HBuDjgc',
76 'ext': 'mp4',
77 'title': 'Douglas Adams: Parrots the Universe and Everything',
78 'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
79 'uploader': 'University of California Television (UCTV)',
80 'uploader_id': 'UCtelevision',
81 'upload_date': '20080522',
82 },
83 'params': {
84 'skip_download': True,
85 },
86 }]
87
88 _NATIVE_FORMATS = {
89 'low': {'preference': 1, 'width': 320, 'height': 180},
90 'medium': {'preference': 2, 'width': 512, 'height': 288},
91 'high': {'preference': 3, 'width': 854, 'height': 480},
92 }
93
94 def _extract_info(self, webpage):
95 info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
96 webpage, 'info json')
97 return json.loads(info_json)
98
99 def _real_extract(self, url):
100 m = re.match(self._VALID_URL, url, re.VERBOSE)
101 if m.group('type') == 'embed':
102 desktop_url = m.group('proto') + 'www' + m.group('urlmain')
103 return self.url_result(desktop_url, 'TED')
104 name = m.group('name')
105 if m.group('type_talk'):
106 return self._talk_info(url, name)
107 elif m.group('type_watch'):
108 return self._watch_info(url, name)
109 else:
110 return self._playlist_videos_info(url, name)
111
112 def _playlist_videos_info(self, url, name):
113 '''Returns the videos of the playlist'''
114
115 webpage = self._download_webpage(url, name,
116 'Downloading playlist webpage')
117 info = self._extract_info(webpage)
118 playlist_info = info['playlist']
119
120 playlist_entries = [
121 self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
122 for talk in info['talks']
123 ]
124 return self.playlist_result(
125 playlist_entries,
126 playlist_id=compat_str(playlist_info['id']),
127 playlist_title=playlist_info['title'])
128
129 def _talk_info(self, url, video_name):
130 webpage = self._download_webpage(url, video_name)
131 self.report_extraction(video_name)
132
133 talk_info = self._extract_info(webpage)['talks'][0]
134
135 if talk_info.get('external') is not None:
136 self.to_screen('Found video from %s' % talk_info['external']['service'])
137 return {
138 '_type': 'url',
139 'url': talk_info['external']['uri'],
140 }
141
142 formats = [{
143 'url': format_url,
144 'format_id': format_id,
145 'format': format_id,
146 } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
147 if formats:
148 for f in formats:
149 finfo = self._NATIVE_FORMATS.get(f['format_id'])
150 if finfo:
151 f.update(finfo)
152 else:
153 # Use rtmp downloads
154 formats = [{
155 'format_id': f['name'],
156 'url': talk_info['streamer'],
157 'play_path': f['file'],
158 'ext': 'flv',
159 'width': f['width'],
160 'height': f['height'],
161 'tbr': f['bitrate'],
162 } for f in talk_info['resources']['rtmp']]
163 self._sort_formats(formats)
164
165 video_id = compat_str(talk_info['id'])
166 # subtitles
167 video_subtitles = self.extract_subtitles(video_id, talk_info)
168 if self._downloader.params.get('listsubtitles', False):
169 self._list_available_subtitles(video_id, talk_info)
170 return
171
172 thumbnail = talk_info['thumb']
173 if not thumbnail.startswith('http'):
174 thumbnail = 'http://' + thumbnail
175 return {
176 'id': video_id,
177 'title': talk_info['title'].strip(),
178 'uploader': talk_info['speaker'],
179 'thumbnail': thumbnail,
180 'description': self._og_search_description(webpage),
181 'subtitles': video_subtitles,
182 'formats': formats,
183 'duration': talk_info.get('duration'),
184 }
185
186 def _get_available_subtitles(self, video_id, talk_info):
187 languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
188 if languages:
189 sub_lang_list = {}
190 for l in languages:
191 url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
192 sub_lang_list[l] = url
193 return sub_lang_list
194 else:
195 self._downloader.report_warning('video doesn\'t have subtitles')
196 return {}
197
198 def _watch_info(self, url, name):
199 webpage = self._download_webpage(url, name)
200
201 config_json = self._html_search_regex(
202 r"data-config='([^']+)", webpage, 'config')
203 config = json.loads(config_json)
204 video_url = config['video']['url']
205 thumbnail = config.get('image', {}).get('url')
206
207 title = self._html_search_regex(
208 r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
209 description = self._html_search_regex(
210 [
211 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
212 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
213 ],
214 webpage, 'description', fatal=False)
215
216 return {
217 'id': name,
218 'url': video_url,
219 'title': title,
220 'thumbnail': thumbnail,
221 'description': description,
222 }