]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/frontendmasters.py
New upstream version 2019.01.16
[youtubedl] / youtube_dl / extractor / frontendmasters.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import (
8 compat_str,
9 compat_urlparse,
10 )
11 from ..utils import (
12 ExtractorError,
13 parse_duration,
14 url_or_none,
15 urlencode_postdata,
16 )
17
18
19 class FrontendMastersBaseIE(InfoExtractor):
20 _API_BASE = 'https://api.frontendmasters.com/v1/kabuki'
21 _LOGIN_URL = 'https://frontendmasters.com/login/'
22
23 _NETRC_MACHINE = 'frontendmasters'
24
25 _QUALITIES = {
26 'low': {'width': 480, 'height': 360},
27 'mid': {'width': 1280, 'height': 720},
28 'high': {'width': 1920, 'height': 1080}
29 }
30
31 def _real_initialize(self):
32 self._login()
33
34 def _login(self):
35 (username, password) = self._get_login_info()
36 if username is None:
37 return
38
39 login_page = self._download_webpage(
40 self._LOGIN_URL, None, 'Downloading login page')
41
42 login_form = self._hidden_inputs(login_page)
43
44 login_form.update({
45 'username': username,
46 'password': password
47 })
48
49 post_url = self._search_regex(
50 r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
51 'post_url', default=self._LOGIN_URL, group='url')
52
53 if not post_url.startswith('http'):
54 post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
55
56 response = self._download_webpage(
57 post_url, None, 'Logging in', data=urlencode_postdata(login_form),
58 headers={'Content-Type': 'application/x-www-form-urlencoded'})
59
60 # Successful login
61 if any(p in response for p in (
62 'wp-login.php?action=logout', '>Logout')):
63 return
64
65 error = self._html_search_regex(
66 r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P<error>[^<]+)<',
67 response, 'error message', default=None, group='error')
68 if error:
69 raise ExtractorError('Unable to login: %s' % error, expected=True)
70 raise ExtractorError('Unable to log in')
71
72
73 class FrontendMastersPageBaseIE(FrontendMastersBaseIE):
74 def _download_course(self, course_name, url):
75 return self._download_json(
76 '%s/courses/%s' % (self._API_BASE, course_name), course_name,
77 'Downloading course JSON', headers={'Referer': url})
78
79 @staticmethod
80 def _extract_chapters(course):
81 chapters = []
82 lesson_elements = course.get('lessonElements')
83 if isinstance(lesson_elements, list):
84 chapters = [url_or_none(e) for e in lesson_elements if url_or_none(e)]
85 return chapters
86
87 @staticmethod
88 def _extract_lesson(chapters, lesson_id, lesson):
89 title = lesson.get('title') or lesson_id
90 display_id = lesson.get('slug')
91 description = lesson.get('description')
92 thumbnail = lesson.get('thumbnail')
93
94 chapter_number = None
95 index = lesson.get('index')
96 element_index = lesson.get('elementIndex')
97 if (isinstance(index, int) and isinstance(element_index, int) and
98 index < element_index):
99 chapter_number = element_index - index
100 chapter = (chapters[chapter_number - 1]
101 if chapter_number - 1 < len(chapters) else None)
102
103 duration = None
104 timestamp = lesson.get('timestamp')
105 if isinstance(timestamp, compat_str):
106 mobj = re.search(
107 r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P<end>\s*\d{1,2}:\d{1,2}:\d{1,2})',
108 timestamp)
109 if mobj:
110 duration = parse_duration(mobj.group('end')) - parse_duration(
111 mobj.group('start'))
112
113 return {
114 '_type': 'url_transparent',
115 'url': 'frontendmasters:%s' % lesson_id,
116 'ie_key': FrontendMastersIE.ie_key(),
117 'id': lesson_id,
118 'display_id': display_id,
119 'title': title,
120 'description': description,
121 'thumbnail': thumbnail,
122 'duration': duration,
123 'chapter': chapter,
124 'chapter_number': chapter_number,
125 }
126
127
128 class FrontendMastersIE(FrontendMastersBaseIE):
129 _VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)'
130 _TESTS = [{
131 'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba',
132 'md5': '7f161159710d6b7016a4f4af6fcb05e2',
133 'info_dict': {
134 'id': 'a2qogef6ba',
135 'ext': 'mp4',
136 'title': 'a2qogef6ba',
137 },
138 'skip': 'Requires FrontendMasters account credentials',
139 }, {
140 'url': 'frontendmasters:a2qogef6ba',
141 'only_matching': True,
142 }]
143
144 def _real_extract(self, url):
145 lesson_id = self._match_id(url)
146
147 source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id)
148
149 formats = []
150 for ext in ('webm', 'mp4'):
151 for quality in ('low', 'mid', 'high'):
152 resolution = self._QUALITIES[quality].copy()
153 format_id = '%s-%s' % (ext, quality)
154 format_url = self._download_json(
155 source_url, lesson_id,
156 'Downloading %s source JSON' % format_id, query={
157 'f': ext,
158 'r': resolution['height'],
159 }, headers={
160 'Referer': url,
161 }, fatal=False)['url']
162
163 if not format_url:
164 continue
165
166 f = resolution.copy()
167 f.update({
168 'url': format_url,
169 'ext': ext,
170 'format_id': format_id,
171 })
172 formats.append(f)
173 self._sort_formats(formats)
174
175 subtitles = {
176 'en': [{
177 'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id),
178 }]
179 }
180
181 return {
182 'id': lesson_id,
183 'title': lesson_id,
184 'formats': formats,
185 'subtitles': subtitles
186 }
187
188
189 class FrontendMastersLessonIE(FrontendMastersPageBaseIE):
190 _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)'
191 _TEST = {
192 'url': 'https://frontendmasters.com/courses/web-development/tools',
193 'info_dict': {
194 'id': 'a2qogef6ba',
195 'display_id': 'tools',
196 'ext': 'mp4',
197 'title': 'Tools',
198 'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7',
199 'thumbnail': r're:^https?://.*\.jpg$',
200 'chapter': 'Introduction',
201 'chapter_number': 1,
202 },
203 'params': {
204 'skip_download': True,
205 },
206 'skip': 'Requires FrontendMasters account credentials',
207 }
208
209 def _real_extract(self, url):
210 mobj = re.match(self._VALID_URL, url)
211 course_name, lesson_name = mobj.group('course_name', 'lesson_name')
212
213 course = self._download_course(course_name, url)
214
215 lesson_id, lesson = next(
216 (video_id, data)
217 for video_id, data in course['lessonData'].items()
218 if data.get('slug') == lesson_name)
219
220 chapters = self._extract_chapters(course)
221 return self._extract_lesson(chapters, lesson_id, lesson)
222
223
224 class FrontendMastersCourseIE(FrontendMastersPageBaseIE):
225 _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)'
226 _TEST = {
227 'url': 'https://frontendmasters.com/courses/web-development/',
228 'info_dict': {
229 'id': 'web-development',
230 'title': 'Introduction to Web Development',
231 'description': 'md5:9317e6e842098bf725d62360e52d49a6',
232 },
233 'playlist_count': 81,
234 'skip': 'Requires FrontendMasters account credentials',
235 }
236
237 @classmethod
238 def suitable(cls, url):
239 return False if FrontendMastersLessonIE.suitable(url) else super(
240 FrontendMastersBaseIE, cls).suitable(url)
241
242 def _real_extract(self, url):
243 course_name = self._match_id(url)
244
245 course = self._download_course(course_name, url)
246
247 chapters = self._extract_chapters(course)
248
249 lessons = sorted(
250 course['lessonData'].values(), key=lambda data: data['index'])
251
252 entries = []
253 for lesson in lessons:
254 lesson_name = lesson.get('slug')
255 if not lesson_name:
256 continue
257 lesson_id = lesson.get('hash') or lesson.get('statsId')
258 entries.append(self._extract_lesson(chapters, lesson_id, lesson))
259
260 title = course.get('title')
261 description = course.get('description')
262
263 return self.playlist_result(entries, course_name, title, description)