Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/lecturio.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import compat_str
   8 from ..utils import (
   9     clean_html,
  10     determine_ext,
  11     ExtractorError,
  12     float_or_none,
  13     int_or_none,
  14     str_or_none,
  15     url_or_none,
  16     urlencode_postdata,
  17     urljoin,
  18 )
  19
  20
  21 class LecturioBaseIE(InfoExtractor):
  22     _API_BASE_URL = 'https://app.lecturio.com/api/en/latest/html5/'
  23     _LOGIN_URL = 'https://app.lecturio.com/en/login'
  24     _NETRC_MACHINE = 'lecturio'
  25
  26     def _real_initialize(self):
  27         self._login()
  28
  29     def _login(self):
  30         username, password = self._get_login_info()
  31         if username is None:
  32             return
  33
  34         # Sets some cookies
  35         _, urlh = self._download_webpage_handle(
  36             self._LOGIN_URL, None, 'Downloading login popup')
  37
  38         def is_logged(url_handle):
  39             return self._LOGIN_URL not in compat_str(url_handle.geturl())
  40
  41         # Already logged in
  42         if is_logged(urlh):
  43             return
  44
  45         login_form = {
  46             'signin[email]': username,
  47             'signin[password]': password,
  48             'signin[remember]': 'on',
  49         }
  50
  51         response, urlh = self._download_webpage_handle(
  52             self._LOGIN_URL, None, 'Logging in',
  53             data=urlencode_postdata(login_form))
  54
  55         # Logged in successfully
  56         if is_logged(urlh):
  57             return
  58
  59         errors = self._html_search_regex(
  60             r'(?s)<ul[^>]+class=["\']error_list[^>]+>(.+?)</ul>', response,
  61             'errors', default=None)
  62         if errors:
  63             raise ExtractorError('Unable to login: %s' % errors, expected=True)
  64         raise ExtractorError('Unable to log in')
  65
  66
  67 class LecturioIE(LecturioBaseIE):
  68     _VALID_URL = r'''(?x)
  69                     https://
  70                         (?:
  71                             app\.lecturio\.com/([^/]+/(?P<nt>[^/?#&]+)\.lecture|(?:\#/)?lecture/c/\d+/(?P<id>\d+))|
  72                             (?:www\.)?lecturio\.de/[^/]+/(?P<nt_de>[^/?#&]+)\.vortrag
  73                         )
  74                     '''
  75     _TESTS = [{
  76         'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
  77         'md5': '9a42cf1d8282a6311bf7211bbde26fde',
  78         'info_dict': {
  79             'id': '39634',
  80             'ext': 'mp4',
  81             'title': 'Important Concepts and Terms — Introduction to Microbiology',
  82         },
  83         'skip': 'Requires lecturio account credentials',
  84     }, {
  85         'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
  86         'only_matching': True,
  87     }, {
  88         'url': 'https://app.lecturio.com/#/lecture/c/6434/39634',
  89         'only_matching': True,
  90     }]
  91
  92     _CC_LANGS = {
  93         'Arabic': 'ar',
  94         'Bulgarian': 'bg',
  95         'German': 'de',
  96         'English': 'en',
  97         'Spanish': 'es',
  98         'Persian': 'fa',
  99         'French': 'fr',
 100         'Japanese': 'ja',
 101         'Polish': 'pl',
 102         'Pashto': 'ps',
 103         'Russian': 'ru',
 104     }
 105
 106     def _real_extract(self, url):
 107         mobj = re.match(self._VALID_URL, url)
 108         nt = mobj.group('nt') or mobj.group('nt_de')
 109         lecture_id = mobj.group('id')
 110         display_id = nt or lecture_id
 111         api_path = 'lectures/' + lecture_id if lecture_id else 'lecture/' + nt + '.json'
 112         video = self._download_json(
 113             self._API_BASE_URL + api_path, display_id)
 114         title = video['title'].strip()
 115         if not lecture_id:
 116             pid = video.get('productId') or video.get('uid')
 117             if pid:
 118                 spid = pid.split('_')
 119                 if spid and len(spid) == 2:
 120                     lecture_id = spid[1]
 121
 122         formats = []
 123         for format_ in video['content']['media']:
 124             if not isinstance(format_, dict):
 125                 continue
 126             file_ = format_.get('file')
 127             if not file_:
 128                 continue
 129             ext = determine_ext(file_)
 130             if ext == 'smil':
 131                 # smil contains only broken RTMP formats anyway
 132                 continue
 133             file_url = url_or_none(file_)
 134             if not file_url:
 135                 continue
 136             label = str_or_none(format_.get('label'))
 137             filesize = int_or_none(format_.get('fileSize'))
 138             f = {
 139                 'url': file_url,
 140                 'format_id': label,
 141                 'filesize': float_or_none(filesize, invscale=1000)
 142             }
 143             if label:
 144                 mobj = re.match(r'(\d+)p\s*\(([^)]+)\)', label)
 145                 if mobj:
 146                     f.update({
 147                         'format_id': mobj.group(2),
 148                         'height': int(mobj.group(1)),
 149                     })
 150             formats.append(f)
 151         self._sort_formats(formats)
 152
 153         subtitles = {}
 154         automatic_captions = {}
 155         captions = video.get('captions') or []
 156         for cc in captions:
 157             cc_url = cc.get('url')
 158             if not cc_url:
 159                 continue
 160             cc_label = cc.get('translatedCode')
 161             lang = cc.get('languageCode') or self._search_regex(
 162                 r'/([a-z]{2})_', cc_url, 'lang',
 163                 default=cc_label.split()[0] if cc_label else 'en')
 164             original_lang = self._search_regex(
 165                 r'/[a-z]{2}_([a-z]{2})_', cc_url, 'original lang',
 166                 default=None)
 167             sub_dict = (automatic_captions
 168                         if 'auto-translated' in cc_label or original_lang
 169                         else subtitles)
 170             sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
 171                 'url': cc_url,
 172             })
 173
 174         return {
 175             'id': lecture_id or nt,
 176             'title': title,
 177             'formats': formats,
 178             'subtitles': subtitles,
 179             'automatic_captions': automatic_captions,
 180         }
 181
 182
 183 class LecturioCourseIE(LecturioBaseIE):
 184     _VALID_URL = r'https://app\.lecturio\.com/(?:[^/]+/(?P<nt>[^/?#&]+)\.course|(?:#/)?course/c/(?P<id>\d+))'
 185     _TESTS = [{
 186         'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
 187         'info_dict': {
 188             'id': 'microbiology-introduction',
 189             'title': 'Microbiology: Introduction',
 190             'description': 'md5:13da8500c25880c6016ae1e6d78c386a',
 191         },
 192         'playlist_count': 45,
 193         'skip': 'Requires lecturio account credentials',
 194     }, {
 195         'url': 'https://app.lecturio.com/#/course/c/6434',
 196         'only_matching': True,
 197     }]
 198
 199     def _real_extract(self, url):
 200         nt, course_id = re.match(self._VALID_URL, url).groups()
 201         display_id = nt or course_id
 202         api_path = 'courses/' + course_id if course_id else 'course/content/' + nt + '.json'
 203         course = self._download_json(
 204             self._API_BASE_URL + api_path, display_id)
 205         entries = []
 206         for lecture in course.get('lectures', []):
 207             lecture_id = str_or_none(lecture.get('id'))
 208             lecture_url = lecture.get('url')
 209             if lecture_url:
 210                 lecture_url = urljoin(url, lecture_url)
 211             else:
 212                 lecture_url = 'https://app.lecturio.com/#/lecture/c/%s/%s' % (course_id, lecture_id)
 213             entries.append(self.url_result(
 214                 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
 215         return self.playlist_result(
 216             entries, display_id, course.get('title'),
 217             clean_html(course.get('description')))
 218
 219
 220 class LecturioDeCourseIE(LecturioBaseIE):
 221     _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
 222     _TEST = {
 223         'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
 224         'only_matching': True,
 225     }
 226
 227     def _real_extract(self, url):
 228         display_id = self._match_id(url)
 229
 230         webpage = self._download_webpage(url, display_id)
 231
 232         entries = []
 233         for mobj in re.finditer(
 234                 r'(?s)<td[^>]+\bdata-lecture-id=["\'](?P<id>\d+).+?\bhref=(["\'])(?P<url>(?:(?!\2).)+\.vortrag)\b[^>]+>',
 235                 webpage):
 236             lecture_url = urljoin(url, mobj.group('url'))
 237             lecture_id = mobj.group('id')
 238             entries.append(self.url_result(
 239                 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
 240
 241         title = self._search_regex(
 242             r'<h1[^>]*>([^<]+)', webpage, 'title', default=None)
 243
 244         return self.playlist_result(entries, display_id, title)