]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/lecturio.py
Update upstream source from tag 'upstream/2019.09.28'
[youtubedl] / youtube_dl / extractor / lecturio.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_str
8 from ..utils import (
9 clean_html,
10 determine_ext,
11 ExtractorError,
12 float_or_none,
13 int_or_none,
14 str_or_none,
15 url_or_none,
16 urlencode_postdata,
17 urljoin,
18 )
19
20
21 class LecturioBaseIE(InfoExtractor):
22 _API_BASE_URL = 'https://app.lecturio.com/api/en/latest/html5/'
23 _LOGIN_URL = 'https://app.lecturio.com/en/login'
24 _NETRC_MACHINE = 'lecturio'
25
26 def _real_initialize(self):
27 self._login()
28
29 def _login(self):
30 username, password = self._get_login_info()
31 if username is None:
32 return
33
34 # Sets some cookies
35 _, urlh = self._download_webpage_handle(
36 self._LOGIN_URL, None, 'Downloading login popup')
37
38 def is_logged(url_handle):
39 return self._LOGIN_URL not in compat_str(url_handle.geturl())
40
41 # Already logged in
42 if is_logged(urlh):
43 return
44
45 login_form = {
46 'signin[email]': username,
47 'signin[password]': password,
48 'signin[remember]': 'on',
49 }
50
51 response, urlh = self._download_webpage_handle(
52 self._LOGIN_URL, None, 'Logging in',
53 data=urlencode_postdata(login_form))
54
55 # Logged in successfully
56 if is_logged(urlh):
57 return
58
59 errors = self._html_search_regex(
60 r'(?s)<ul[^>]+class=["\']error_list[^>]+>(.+?)</ul>', response,
61 'errors', default=None)
62 if errors:
63 raise ExtractorError('Unable to login: %s' % errors, expected=True)
64 raise ExtractorError('Unable to log in')
65
66
67 class LecturioIE(LecturioBaseIE):
68 _VALID_URL = r'''(?x)
69 https://
70 (?:
71 app\.lecturio\.com/([^/]+/(?P<nt>[^/?#&]+)\.lecture|(?:\#/)?lecture/c/\d+/(?P<id>\d+))|
72 (?:www\.)?lecturio\.de/[^/]+/(?P<nt_de>[^/?#&]+)\.vortrag
73 )
74 '''
75 _TESTS = [{
76 'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
77 'md5': '9a42cf1d8282a6311bf7211bbde26fde',
78 'info_dict': {
79 'id': '39634',
80 'ext': 'mp4',
81 'title': 'Important Concepts and Terms — Introduction to Microbiology',
82 },
83 'skip': 'Requires lecturio account credentials',
84 }, {
85 'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
86 'only_matching': True,
87 }, {
88 'url': 'https://app.lecturio.com/#/lecture/c/6434/39634',
89 'only_matching': True,
90 }]
91
92 _CC_LANGS = {
93 'Arabic': 'ar',
94 'Bulgarian': 'bg',
95 'German': 'de',
96 'English': 'en',
97 'Spanish': 'es',
98 'Persian': 'fa',
99 'French': 'fr',
100 'Japanese': 'ja',
101 'Polish': 'pl',
102 'Pashto': 'ps',
103 'Russian': 'ru',
104 }
105
106 def _real_extract(self, url):
107 mobj = re.match(self._VALID_URL, url)
108 nt = mobj.group('nt') or mobj.group('nt_de')
109 lecture_id = mobj.group('id')
110 display_id = nt or lecture_id
111 api_path = 'lectures/' + lecture_id if lecture_id else 'lecture/' + nt + '.json'
112 video = self._download_json(
113 self._API_BASE_URL + api_path, display_id)
114 title = video['title'].strip()
115 if not lecture_id:
116 pid = video.get('productId') or video.get('uid')
117 if pid:
118 spid = pid.split('_')
119 if spid and len(spid) == 2:
120 lecture_id = spid[1]
121
122 formats = []
123 for format_ in video['content']['media']:
124 if not isinstance(format_, dict):
125 continue
126 file_ = format_.get('file')
127 if not file_:
128 continue
129 ext = determine_ext(file_)
130 if ext == 'smil':
131 # smil contains only broken RTMP formats anyway
132 continue
133 file_url = url_or_none(file_)
134 if not file_url:
135 continue
136 label = str_or_none(format_.get('label'))
137 filesize = int_or_none(format_.get('fileSize'))
138 f = {
139 'url': file_url,
140 'format_id': label,
141 'filesize': float_or_none(filesize, invscale=1000)
142 }
143 if label:
144 mobj = re.match(r'(\d+)p\s*\(([^)]+)\)', label)
145 if mobj:
146 f.update({
147 'format_id': mobj.group(2),
148 'height': int(mobj.group(1)),
149 })
150 formats.append(f)
151 self._sort_formats(formats)
152
153 subtitles = {}
154 automatic_captions = {}
155 captions = video.get('captions') or []
156 for cc in captions:
157 cc_url = cc.get('url')
158 if not cc_url:
159 continue
160 cc_label = cc.get('translatedCode')
161 lang = cc.get('languageCode') or self._search_regex(
162 r'/([a-z]{2})_', cc_url, 'lang',
163 default=cc_label.split()[0] if cc_label else 'en')
164 original_lang = self._search_regex(
165 r'/[a-z]{2}_([a-z]{2})_', cc_url, 'original lang',
166 default=None)
167 sub_dict = (automatic_captions
168 if 'auto-translated' in cc_label or original_lang
169 else subtitles)
170 sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
171 'url': cc_url,
172 })
173
174 return {
175 'id': lecture_id or nt,
176 'title': title,
177 'formats': formats,
178 'subtitles': subtitles,
179 'automatic_captions': automatic_captions,
180 }
181
182
183 class LecturioCourseIE(LecturioBaseIE):
184 _VALID_URL = r'https://app\.lecturio\.com/(?:[^/]+/(?P<nt>[^/?#&]+)\.course|(?:#/)?course/c/(?P<id>\d+))'
185 _TESTS = [{
186 'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
187 'info_dict': {
188 'id': 'microbiology-introduction',
189 'title': 'Microbiology: Introduction',
190 'description': 'md5:13da8500c25880c6016ae1e6d78c386a',
191 },
192 'playlist_count': 45,
193 'skip': 'Requires lecturio account credentials',
194 }, {
195 'url': 'https://app.lecturio.com/#/course/c/6434',
196 'only_matching': True,
197 }]
198
199 def _real_extract(self, url):
200 nt, course_id = re.match(self._VALID_URL, url).groups()
201 display_id = nt or course_id
202 api_path = 'courses/' + course_id if course_id else 'course/content/' + nt + '.json'
203 course = self._download_json(
204 self._API_BASE_URL + api_path, display_id)
205 entries = []
206 for lecture in course.get('lectures', []):
207 lecture_id = str_or_none(lecture.get('id'))
208 lecture_url = lecture.get('url')
209 if lecture_url:
210 lecture_url = urljoin(url, lecture_url)
211 else:
212 lecture_url = 'https://app.lecturio.com/#/lecture/c/%s/%s' % (course_id, lecture_id)
213 entries.append(self.url_result(
214 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
215 return self.playlist_result(
216 entries, display_id, course.get('title'),
217 clean_html(course.get('description')))
218
219
220 class LecturioDeCourseIE(LecturioBaseIE):
221 _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
222 _TEST = {
223 'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
224 'only_matching': True,
225 }
226
227 def _real_extract(self, url):
228 display_id = self._match_id(url)
229
230 webpage = self._download_webpage(url, display_id)
231
232 entries = []
233 for mobj in re.finditer(
234 r'(?s)<td[^>]+\bdata-lecture-id=["\'](?P<id>\d+).+?\bhref=(["\'])(?P<url>(?:(?!\2).)+\.vortrag)\b[^>]+>',
235 webpage):
236 lecture_url = urljoin(url, mobj.group('url'))
237 lecture_id = mobj.group('id')
238 entries.append(self.url_result(
239 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
240
241 title = self._search_regex(
242 r'<h1[^>]*>([^<]+)', webpage, 'title', default=None)
243
244 return self.playlist_result(entries, display_id, title)