]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/lecturio.py
Update upstream source from tag 'upstream/2019.06.08'
[youtubedl] / youtube_dl / extractor / lecturio.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_str
8 from ..utils import (
9 determine_ext,
10 extract_attributes,
11 ExtractorError,
12 float_or_none,
13 int_or_none,
14 str_or_none,
15 url_or_none,
16 urlencode_postdata,
17 urljoin,
18 )
19
20
21 class LecturioBaseIE(InfoExtractor):
22 _LOGIN_URL = 'https://app.lecturio.com/en/login'
23 _NETRC_MACHINE = 'lecturio'
24
25 def _real_initialize(self):
26 self._login()
27
28 def _login(self):
29 username, password = self._get_login_info()
30 if username is None:
31 return
32
33 # Sets some cookies
34 _, urlh = self._download_webpage_handle(
35 self._LOGIN_URL, None, 'Downloading login popup')
36
37 def is_logged(url_handle):
38 return self._LOGIN_URL not in compat_str(url_handle.geturl())
39
40 # Already logged in
41 if is_logged(urlh):
42 return
43
44 login_form = {
45 'signin[email]': username,
46 'signin[password]': password,
47 'signin[remember]': 'on',
48 }
49
50 response, urlh = self._download_webpage_handle(
51 self._LOGIN_URL, None, 'Logging in',
52 data=urlencode_postdata(login_form))
53
54 # Logged in successfully
55 if is_logged(urlh):
56 return
57
58 errors = self._html_search_regex(
59 r'(?s)<ul[^>]+class=["\']error_list[^>]+>(.+?)</ul>', response,
60 'errors', default=None)
61 if errors:
62 raise ExtractorError('Unable to login: %s' % errors, expected=True)
63 raise ExtractorError('Unable to log in')
64
65
66 class LecturioIE(LecturioBaseIE):
67 _VALID_URL = r'''(?x)
68 https://
69 (?:
70 app\.lecturio\.com/[^/]+/(?P<id>[^/?#&]+)\.lecture|
71 (?:www\.)?lecturio\.de/[^/]+/(?P<id_de>[^/?#&]+)\.vortrag
72 )
73 '''
74 _TESTS = [{
75 'url': 'https://app.lecturio.com/medical-courses/important-concepts-and-terms-introduction-to-microbiology.lecture#tab/videos',
76 'md5': 'f576a797a5b7a5e4e4bbdfc25a6a6870',
77 'info_dict': {
78 'id': '39634',
79 'ext': 'mp4',
80 'title': 'Important Concepts and Terms – Introduction to Microbiology',
81 },
82 'skip': 'Requires lecturio account credentials',
83 }, {
84 'url': 'https://www.lecturio.de/jura/oeffentliches-recht-staatsexamen.vortrag',
85 'only_matching': True,
86 }]
87
88 _CC_LANGS = {
89 'German': 'de',
90 'English': 'en',
91 'Spanish': 'es',
92 'French': 'fr',
93 'Polish': 'pl',
94 'Russian': 'ru',
95 }
96
97 def _real_extract(self, url):
98 mobj = re.match(self._VALID_URL, url)
99 display_id = mobj.group('id') or mobj.group('id_de')
100
101 webpage = self._download_webpage(
102 'https://app.lecturio.com/en/lecture/%s/player.html' % display_id,
103 display_id)
104
105 lecture_id = self._search_regex(
106 r'lecture_id\s*=\s*(?:L_)?(\d+)', webpage, 'lecture id')
107
108 api_url = self._search_regex(
109 r'lectureDataLink\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
110 'api url', group='url')
111
112 video = self._download_json(api_url, display_id)
113
114 title = video['title'].strip()
115
116 formats = []
117 for format_ in video['content']['media']:
118 if not isinstance(format_, dict):
119 continue
120 file_ = format_.get('file')
121 if not file_:
122 continue
123 ext = determine_ext(file_)
124 if ext == 'smil':
125 # smil contains only broken RTMP formats anyway
126 continue
127 file_url = url_or_none(file_)
128 if not file_url:
129 continue
130 label = str_or_none(format_.get('label'))
131 filesize = int_or_none(format_.get('fileSize'))
132 formats.append({
133 'url': file_url,
134 'format_id': label,
135 'filesize': float_or_none(filesize, invscale=1000)
136 })
137 self._sort_formats(formats)
138
139 subtitles = {}
140 automatic_captions = {}
141 cc = self._parse_json(
142 self._search_regex(
143 r'subtitleUrls\s*:\s*({.+?})\s*,', webpage, 'subtitles',
144 default='{}'), display_id, fatal=False)
145 for cc_label, cc_url in cc.items():
146 cc_url = url_or_none(cc_url)
147 if not cc_url:
148 continue
149 lang = self._search_regex(
150 r'/([a-z]{2})_', cc_url, 'lang',
151 default=cc_label.split()[0] if cc_label else 'en')
152 original_lang = self._search_regex(
153 r'/[a-z]{2}_([a-z]{2})_', cc_url, 'original lang',
154 default=None)
155 sub_dict = (automatic_captions
156 if 'auto-translated' in cc_label or original_lang
157 else subtitles)
158 sub_dict.setdefault(self._CC_LANGS.get(lang, lang), []).append({
159 'url': cc_url,
160 })
161
162 return {
163 'id': lecture_id,
164 'title': title,
165 'formats': formats,
166 'subtitles': subtitles,
167 'automatic_captions': automatic_captions,
168 }
169
170
171 class LecturioCourseIE(LecturioBaseIE):
172 _VALID_URL = r'https://app\.lecturio\.com/[^/]+/(?P<id>[^/?#&]+)\.course'
173 _TEST = {
174 'url': 'https://app.lecturio.com/medical-courses/microbiology-introduction.course#/',
175 'info_dict': {
176 'id': 'microbiology-introduction',
177 'title': 'Microbiology: Introduction',
178 },
179 'playlist_count': 45,
180 'skip': 'Requires lecturio account credentials',
181 }
182
183 def _real_extract(self, url):
184 display_id = self._match_id(url)
185
186 webpage = self._download_webpage(url, display_id)
187
188 entries = []
189 for mobj in re.finditer(
190 r'(?s)<[^>]+\bdata-url=(["\'])(?:(?!\1).)+\.lecture\b[^>]+>',
191 webpage):
192 params = extract_attributes(mobj.group(0))
193 lecture_url = urljoin(url, params.get('data-url'))
194 lecture_id = params.get('data-id')
195 entries.append(self.url_result(
196 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
197
198 title = self._search_regex(
199 r'<span[^>]+class=["\']content-title[^>]+>([^<]+)', webpage,
200 'title', default=None)
201
202 return self.playlist_result(entries, display_id, title)
203
204
205 class LecturioDeCourseIE(LecturioBaseIE):
206 _VALID_URL = r'https://(?:www\.)?lecturio\.de/[^/]+/(?P<id>[^/?#&]+)\.kurs'
207 _TEST = {
208 'url': 'https://www.lecturio.de/jura/grundrechte.kurs',
209 'only_matching': True,
210 }
211
212 def _real_extract(self, url):
213 display_id = self._match_id(url)
214
215 webpage = self._download_webpage(url, display_id)
216
217 entries = []
218 for mobj in re.finditer(
219 r'(?s)<td[^>]+\bdata-lecture-id=["\'](?P<id>\d+).+?\bhref=(["\'])(?P<url>(?:(?!\2).)+\.vortrag)\b[^>]+>',
220 webpage):
221 lecture_url = urljoin(url, mobj.group('url'))
222 lecture_id = mobj.group('id')
223 entries.append(self.url_result(
224 lecture_url, ie=LecturioIE.ie_key(), video_id=lecture_id))
225
226 title = self._search_regex(
227 r'<h1[^>]*>([^<]+)', webpage, 'title', default=None)
228
229 return self.playlist_result(entries, display_id, title)