Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ted.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import re
   5
   6 from .subtitles import SubtitlesInfoExtractor
   7
   8 from ..utils import (
   9     compat_str,
  10 )
  11
  12
  13 class TEDIE(SubtitlesInfoExtractor):
  14     _VALID_URL = r'''(?x)
  15         (?P<proto>https?://)
  16         (?P<type>www|embed)(?P<urlmain>\.ted\.com/
  17         (
  18             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
  19             |
  20             ((?P<type_talk>talks)) # We have a simple talk
  21             |
  22             (?P<type_watch>watch)/[^/]+/[^/]+
  23         )
  24         (/lang/(.*?))? # The url may contain the language
  25         /(?P<name>[\w-]+) # Here goes the name and then ".html"
  26         .*)$
  27         '''
  28     _TESTS = [{
  29         'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
  30         'md5': 'fc94ac279feebbce69f21c0c6ee82810',
  31         'info_dict': {
  32             'id': '102',
  33             'ext': 'mp4',
  34             'title': 'The illusion of consciousness',
  35             'description': ('Philosopher Dan Dennett makes a compelling '
  36                 'argument that not only don\'t we understand our own '
  37                 'consciousness, but that half the time our brains are '
  38                 'actively fooling us.'),
  39             'uploader': 'Dan Dennett',
  40             'width': 854,
  41         }
  42     }, {
  43         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
  44         'md5': '226f4fb9c62380d11b7995efa4c87994',
  45         'info_dict': {
  46             'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
  47             'ext': 'mp4',
  48             'title': 'Vishal Sikka: The beauty and power of algorithms',
  49             'thumbnail': 're:^https?://.+\.jpg',
  50             'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
  51         }
  52     }, {
  53         'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
  54         'info_dict': {
  55             'id': '1972',
  56             'ext': 'mp4',
  57             'title': 'Be passionate. Be courageous. Be your best.',
  58             'uploader': 'Gabby Giffords and Mark Kelly',
  59             'description': 'md5:5174aed4d0f16021b704120360f72b92',
  60         },
  61     }, {
  62         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
  63         'info_dict': {
  64             'id': '10',
  65             'title': 'Who are the hackers?',
  66         },
  67         'playlist_mincount': 6,
  68     }, {
  69         # contains a youtube video
  70         'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
  71         'add_ie': ['Youtube'],
  72         'info_dict': {
  73             'id': '_ZG8HBuDjgc',
  74             'ext': 'mp4',
  75             'title': 'Douglas Adams: Parrots the Universe and Everything',
  76             'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
  77             'uploader': 'University of California Television (UCTV)',
  78             'uploader_id': 'UCtelevision',
  79             'upload_date': '20080522',
  80         },
  81         'params': {
  82             'skip_download': True,
  83         },
  84     }]
  85
  86     _NATIVE_FORMATS = {
  87         'low': {'preference': 1, 'width': 320, 'height': 180},
  88         'medium': {'preference': 2, 'width': 512, 'height': 288},
  89         'high': {'preference': 3, 'width': 854, 'height': 480},
  90     }
  91
  92     def _extract_info(self, webpage):
  93         info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
  94             webpage, 'info json')
  95         return json.loads(info_json)
  96
  97     def _real_extract(self, url):
  98         m = re.match(self._VALID_URL, url, re.VERBOSE)
  99         if m.group('type') == 'embed':
 100             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
 101             return self.url_result(desktop_url, 'TED')
 102         name = m.group('name')
 103         if m.group('type_talk'):
 104             return self._talk_info(url, name)
 105         elif m.group('type_watch'):
 106             return self._watch_info(url, name)
 107         else:
 108             return self._playlist_videos_info(url, name)
 109
 110     def _playlist_videos_info(self, url, name):
 111         '''Returns the videos of the playlist'''
 112
 113         webpage = self._download_webpage(url, name,
 114             'Downloading playlist webpage')
 115         info = self._extract_info(webpage)
 116         playlist_info = info['playlist']
 117
 118         playlist_entries = [
 119             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
 120             for talk in info['talks']
 121         ]
 122         return self.playlist_result(
 123             playlist_entries,
 124             playlist_id=compat_str(playlist_info['id']),
 125             playlist_title=playlist_info['title'])
 126
 127     def _talk_info(self, url, video_name):
 128         webpage = self._download_webpage(url, video_name)
 129         self.report_extraction(video_name)
 130
 131         talk_info = self._extract_info(webpage)['talks'][0]
 132
 133         if talk_info.get('external') is not None:
 134             self.to_screen('Found video from %s' % talk_info['external']['service'])
 135             return {
 136                 '_type': 'url',
 137                 'url': talk_info['external']['uri'],
 138             }
 139
 140         formats = [{
 141             'url': format_url,
 142             'format_id': format_id,
 143             'format': format_id,
 144         } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
 145         if formats:
 146             for f in formats:
 147                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
 148                 if finfo:
 149                     f.update(finfo)
 150         else:
 151             # Use rtmp downloads
 152             formats = [{
 153                 'format_id': f['name'],
 154                 'url': talk_info['streamer'],
 155                 'play_path': f['file'],
 156                 'ext': 'flv',
 157                 'width': f['width'],
 158                 'height': f['height'],
 159                 'tbr': f['bitrate'],
 160             } for f in talk_info['resources']['rtmp']]
 161         self._sort_formats(formats)
 162
 163         video_id = compat_str(talk_info['id'])
 164         # subtitles
 165         video_subtitles = self.extract_subtitles(video_id, talk_info)
 166         if self._downloader.params.get('listsubtitles', False):
 167             self._list_available_subtitles(video_id, talk_info)
 168             return
 169
 170         thumbnail = talk_info['thumb']
 171         if not thumbnail.startswith('http'):
 172             thumbnail = 'http://' + thumbnail
 173         return {
 174             'id': video_id,
 175             'title': talk_info['title'].strip(),
 176             'uploader': talk_info['speaker'],
 177             'thumbnail': thumbnail,
 178             'description': self._og_search_description(webpage),
 179             'subtitles': video_subtitles,
 180             'formats': formats,
 181         }
 182
 183     def _get_available_subtitles(self, video_id, talk_info):
 184         languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
 185         if languages:
 186             sub_lang_list = {}
 187             for l in languages:
 188                 url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
 189                 sub_lang_list[l] = url
 190             return sub_lang_list
 191         else:
 192             self._downloader.report_warning('video doesn\'t have subtitles')
 193             return {}
 194
 195     def _watch_info(self, url, name):
 196         webpage = self._download_webpage(url, name)
 197
 198         config_json = self._html_search_regex(
 199             r"data-config='([^']+)", webpage, 'config')
 200         config = json.loads(config_json)
 201         video_url = config['video']['url']
 202         thumbnail = config.get('image', {}).get('url')
 203
 204         title = self._html_search_regex(
 205             r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
 206         description = self._html_search_regex(
 207             [
 208                 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
 209                 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
 210             ],
 211             webpage, 'description', fatal=False)
 212
 213         return {
 214             'id': name,
 215             'url': video_url,
 216             'title': title,
 217             'thumbnail': thumbnail,
 218             'description': description,
 219         }