Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ted.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import re
   5
   6 from .subtitles import SubtitlesInfoExtractor
   7
   8 from ..utils import (
   9     compat_str,
  10 )
  11
  12
  13 class TEDIE(SubtitlesInfoExtractor):
  14     _VALID_URL = r'''(?x)
  15         (?P<proto>https?://)
  16         (?P<type>www|embed)(?P<urlmain>\.ted\.com/
  17         (
  18             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
  19             |
  20             ((?P<type_talk>talks)) # We have a simple talk
  21             |
  22             (?P<type_watch>watch)/[^/]+/[^/]+
  23         )
  24         (/lang/(.*?))? # The url may contain the language
  25         /(?P<name>[\w-]+) # Here goes the name and then ".html"
  26         .*)$
  27         '''
  28     _TESTS = [{
  29         'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
  30         'md5': 'fc94ac279feebbce69f21c0c6ee82810',
  31         'info_dict': {
  32             'id': '102',
  33             'ext': 'mp4',
  34             'title': 'The illusion of consciousness',
  35             'description': ('Philosopher Dan Dennett makes a compelling '
  36                 'argument that not only don\'t we understand our own '
  37                 'consciousness, but that half the time our brains are '
  38                 'actively fooling us.'),
  39             'uploader': 'Dan Dennett',
  40             'width': 854,
  41             'duration': 1308,
  42         }
  43     }, {
  44         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
  45         'md5': '226f4fb9c62380d11b7995efa4c87994',
  46         'info_dict': {
  47             'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
  48             'ext': 'mp4',
  49             'title': 'Vishal Sikka: The beauty and power of algorithms',
  50             'thumbnail': 're:^https?://.+\.jpg',
  51             'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
  52         }
  53     }, {
  54         'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
  55         'info_dict': {
  56             'id': '1972',
  57             'ext': 'mp4',
  58             'title': 'Be passionate. Be courageous. Be your best.',
  59             'uploader': 'Gabby Giffords and Mark Kelly',
  60             'description': 'md5:5174aed4d0f16021b704120360f72b92',
  61             'duration': 1128,
  62         },
  63     }, {
  64         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
  65         'info_dict': {
  66             'id': '10',
  67             'title': 'Who are the hackers?',
  68         },
  69         'playlist_mincount': 6,
  70     }, {
  71         # contains a youtube video
  72         'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
  73         'add_ie': ['Youtube'],
  74         'info_dict': {
  75             'id': '_ZG8HBuDjgc',
  76             'ext': 'mp4',
  77             'title': 'Douglas Adams: Parrots the Universe and Everything',
  78             'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
  79             'uploader': 'University of California Television (UCTV)',
  80             'uploader_id': 'UCtelevision',
  81             'upload_date': '20080522',
  82         },
  83         'params': {
  84             'skip_download': True,
  85         },
  86     }]
  87
  88     _NATIVE_FORMATS = {
  89         'low': {'preference': 1, 'width': 320, 'height': 180},
  90         'medium': {'preference': 2, 'width': 512, 'height': 288},
  91         'high': {'preference': 3, 'width': 854, 'height': 480},
  92     }
  93
  94     def _extract_info(self, webpage):
  95         info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
  96             webpage, 'info json')
  97         return json.loads(info_json)
  98
  99     def _real_extract(self, url):
 100         m = re.match(self._VALID_URL, url, re.VERBOSE)
 101         if m.group('type') == 'embed':
 102             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
 103             return self.url_result(desktop_url, 'TED')
 104         name = m.group('name')
 105         if m.group('type_talk'):
 106             return self._talk_info(url, name)
 107         elif m.group('type_watch'):
 108             return self._watch_info(url, name)
 109         else:
 110             return self._playlist_videos_info(url, name)
 111
 112     def _playlist_videos_info(self, url, name):
 113         '''Returns the videos of the playlist'''
 114
 115         webpage = self._download_webpage(url, name,
 116             'Downloading playlist webpage')
 117         info = self._extract_info(webpage)
 118         playlist_info = info['playlist']
 119
 120         playlist_entries = [
 121             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
 122             for talk in info['talks']
 123         ]
 124         return self.playlist_result(
 125             playlist_entries,
 126             playlist_id=compat_str(playlist_info['id']),
 127             playlist_title=playlist_info['title'])
 128
 129     def _talk_info(self, url, video_name):
 130         webpage = self._download_webpage(url, video_name)
 131         self.report_extraction(video_name)
 132
 133         talk_info = self._extract_info(webpage)['talks'][0]
 134
 135         if talk_info.get('external') is not None:
 136             self.to_screen('Found video from %s' % talk_info['external']['service'])
 137             return {
 138                 '_type': 'url',
 139                 'url': talk_info['external']['uri'],
 140             }
 141
 142         formats = [{
 143             'url': format_url,
 144             'format_id': format_id,
 145             'format': format_id,
 146         } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
 147         if formats:
 148             for f in formats:
 149                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
 150                 if finfo:
 151                     f.update(finfo)
 152         else:
 153             # Use rtmp downloads
 154             formats = [{
 155                 'format_id': f['name'],
 156                 'url': talk_info['streamer'],
 157                 'play_path': f['file'],
 158                 'ext': 'flv',
 159                 'width': f['width'],
 160                 'height': f['height'],
 161                 'tbr': f['bitrate'],
 162             } for f in talk_info['resources']['rtmp']]
 163         self._sort_formats(formats)
 164
 165         video_id = compat_str(talk_info['id'])
 166         # subtitles
 167         video_subtitles = self.extract_subtitles(video_id, talk_info)
 168         if self._downloader.params.get('listsubtitles', False):
 169             self._list_available_subtitles(video_id, talk_info)
 170             return
 171
 172         thumbnail = talk_info['thumb']
 173         if not thumbnail.startswith('http'):
 174             thumbnail = 'http://' + thumbnail
 175         return {
 176             'id': video_id,
 177             'title': talk_info['title'].strip(),
 178             'uploader': talk_info['speaker'],
 179             'thumbnail': thumbnail,
 180             'description': self._og_search_description(webpage),
 181             'subtitles': video_subtitles,
 182             'formats': formats,
 183             'duration': talk_info.get('duration'),
 184         }
 185
 186     def _get_available_subtitles(self, video_id, talk_info):
 187         languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
 188         if languages:
 189             sub_lang_list = {}
 190             for l in languages:
 191                 url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
 192                 sub_lang_list[l] = url
 193             return sub_lang_list
 194         else:
 195             self._downloader.report_warning('video doesn\'t have subtitles')
 196             return {}
 197
 198     def _watch_info(self, url, name):
 199         webpage = self._download_webpage(url, name)
 200
 201         config_json = self._html_search_regex(
 202             r"data-config='([^']+)", webpage, 'config')
 203         config = json.loads(config_json)
 204         video_url = config['video']['url']
 205         thumbnail = config.get('image', {}).get('url')
 206
 207         title = self._html_search_regex(
 208             r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
 209         description = self._html_search_regex(
 210             [
 211                 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
 212                 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
 213             ],
 214             webpage, 'description', fatal=False)
 215
 216         return {
 217             'id': name,
 218             'url': video_url,
 219             'title': title,
 220             'thumbnail': thumbnail,
 221             'description': description,
 222         }