]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/teachable.py
debian/{compat,control}: Update to debhelper compatibility level 12.
[youtubedl] / youtube_dl / extractor / teachable.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from .wistia import WistiaIE
7 from ..compat import compat_str
8 from ..utils import (
9 clean_html,
10 ExtractorError,
11 get_element_by_class,
12 urlencode_postdata,
13 urljoin,
14 )
15
16
17 class TeachableBaseIE(InfoExtractor):
18 _NETRC_MACHINE = 'teachable'
19 _URL_PREFIX = 'teachable:'
20
21 _SITES = {
22 # Only notable ones here
23 'upskillcourses.com': 'upskill',
24 'academy.gns3.com': 'gns3',
25 'academyhacker.com': 'academyhacker',
26 'stackskills.com': 'stackskills',
27 'market.saleshacker.com': 'saleshacker',
28 'learnability.org': 'learnability',
29 'edurila.com': 'edurila',
30 }
31
32 _VALID_URL_SUB_TUPLE = (_URL_PREFIX, '|'.join(re.escape(site) for site in _SITES.keys()))
33
34 def _real_initialize(self):
35 self._logged_in = False
36
37 def _login(self, site):
38 if self._logged_in:
39 return
40
41 username, password = self._get_login_info(
42 netrc_machine=self._SITES.get(site, site))
43 if username is None:
44 return
45
46 login_page, urlh = self._download_webpage_handle(
47 'https://%s/sign_in' % site, None,
48 'Downloading %s login page' % site)
49
50 login_url = compat_str(urlh.geturl())
51
52 login_form = self._hidden_inputs(login_page)
53
54 login_form.update({
55 'user[email]': username,
56 'user[password]': password,
57 })
58
59 post_url = self._search_regex(
60 r'<form[^>]+action=(["\'])(?P<url>(?:(?!\1).)+)\1', login_page,
61 'post url', default=login_url, group='url')
62
63 if not post_url.startswith('http'):
64 post_url = urljoin(login_url, post_url)
65
66 response = self._download_webpage(
67 post_url, None, 'Logging in to %s' % site,
68 data=urlencode_postdata(login_form),
69 headers={
70 'Content-Type': 'application/x-www-form-urlencoded',
71 'Referer': login_url,
72 })
73
74 if '>I accept the new Privacy Policy<' in response:
75 raise ExtractorError(
76 'Unable to login: %s asks you to accept new Privacy Policy. '
77 'Go to https://%s/ and accept.' % (site, site), expected=True)
78
79 # Successful login
80 if any(re.search(p, response) for p in (
81 r'class=["\']user-signout',
82 r'<a[^>]+\bhref=["\']/sign_out',
83 r'>\s*Log out\s*<')):
84 self._logged_in = True
85 return
86
87 message = get_element_by_class('alert', response)
88 if message is not None:
89 raise ExtractorError(
90 'Unable to login: %s' % clean_html(message), expected=True)
91
92 raise ExtractorError('Unable to log in')
93
94
95 class TeachableIE(TeachableBaseIE):
96 _VALID_URL = r'''(?x)
97 (?:
98 %shttps?://(?P<site_t>[^/]+)|
99 https?://(?:www\.)?(?P<site>%s)
100 )
101 /courses/[^/]+/lectures/(?P<id>\d+)
102 ''' % TeachableBaseIE._VALID_URL_SUB_TUPLE
103
104 _TESTS = [{
105 'url': 'http://upskillcourses.com/courses/essential-web-developer-course/lectures/1747100',
106 'info_dict': {
107 'id': 'uzw6zw58or',
108 'ext': 'mp4',
109 'title': 'Welcome to the Course!',
110 'description': 'md5:65edb0affa582974de4625b9cdea1107',
111 'duration': 138.763,
112 'timestamp': 1479846621,
113 'upload_date': '20161122',
114 },
115 'params': {
116 'skip_download': True,
117 },
118 }, {
119 'url': 'http://upskillcourses.com/courses/119763/lectures/1747100',
120 'only_matching': True,
121 }, {
122 'url': 'https://academy.gns3.com/courses/423415/lectures/6885939',
123 'only_matching': True,
124 }, {
125 'url': 'teachable:https://upskillcourses.com/courses/essential-web-developer-course/lectures/1747100',
126 'only_matching': True,
127 }]
128
129 @staticmethod
130 def _is_teachable(webpage):
131 return 'teachableTracker.linker:autoLink' in webpage and re.search(
132 r'<link[^>]+href=["\']https?://process\.fs\.teachablecdn\.com',
133 webpage)
134
135 @staticmethod
136 def _extract_url(webpage, source_url):
137 if not TeachableIE._is_teachable(webpage):
138 return
139 if re.match(r'https?://[^/]+/(?:courses|p)', source_url):
140 return '%s%s' % (TeachableBaseIE._URL_PREFIX, source_url)
141
142 def _real_extract(self, url):
143 mobj = re.match(self._VALID_URL, url)
144 site = mobj.group('site') or mobj.group('site_t')
145 video_id = mobj.group('id')
146
147 self._login(site)
148
149 prefixed = url.startswith(self._URL_PREFIX)
150 if prefixed:
151 url = url[len(self._URL_PREFIX):]
152
153 webpage = self._download_webpage(url, video_id)
154
155 wistia_url = WistiaIE._extract_url(webpage)
156 if not wistia_url:
157 if any(re.search(p, webpage) for p in (
158 r'class=["\']lecture-contents-locked',
159 r'>\s*Lecture contents locked',
160 r'id=["\']lecture-locked')):
161 self.raise_login_required('Lecture contents locked')
162
163 title = self._og_search_title(webpage, default=None)
164
165 return {
166 '_type': 'url_transparent',
167 'url': wistia_url,
168 'ie_key': WistiaIE.ie_key(),
169 'title': title,
170 }
171
172
173 class TeachableCourseIE(TeachableBaseIE):
174 _VALID_URL = r'''(?x)
175 (?:
176 %shttps?://(?P<site_t>[^/]+)|
177 https?://(?:www\.)?(?P<site>%s)
178 )
179 /(?:courses|p)/(?:enrolled/)?(?P<id>[^/?#&]+)
180 ''' % TeachableBaseIE._VALID_URL_SUB_TUPLE
181 _TESTS = [{
182 'url': 'http://upskillcourses.com/courses/essential-web-developer-course/',
183 'info_dict': {
184 'id': 'essential-web-developer-course',
185 'title': 'The Essential Web Developer Course (Free)',
186 },
187 'playlist_count': 192,
188 }, {
189 'url': 'http://upskillcourses.com/courses/119763/',
190 'only_matching': True,
191 }, {
192 'url': 'http://upskillcourses.com/courses/enrolled/119763',
193 'only_matching': True,
194 }, {
195 'url': 'https://academy.gns3.com/courses/enrolled/423415',
196 'only_matching': True,
197 }, {
198 'url': 'teachable:https://learn.vrdev.school/p/gear-vr-developer-mini',
199 'only_matching': True,
200 }, {
201 'url': 'teachable:https://filmsimplified.com/p/davinci-resolve-15-crash-course',
202 'only_matching': True,
203 }]
204
205 @classmethod
206 def suitable(cls, url):
207 return False if TeachableIE.suitable(url) else super(
208 TeachableCourseIE, cls).suitable(url)
209
210 def _real_extract(self, url):
211 mobj = re.match(self._VALID_URL, url)
212 site = mobj.group('site') or mobj.group('site_t')
213 course_id = mobj.group('id')
214
215 self._login(site)
216
217 prefixed = url.startswith(self._URL_PREFIX)
218 if prefixed:
219 prefix = self._URL_PREFIX
220 url = url[len(prefix):]
221
222 webpage = self._download_webpage(url, course_id)
223
224 url_base = 'https://%s/' % site
225
226 entries = []
227
228 for mobj in re.finditer(
229 r'(?s)(?P<li><li[^>]+class=(["\'])(?:(?!\2).)*?section-item[^>]+>.+?</li>)',
230 webpage):
231 li = mobj.group('li')
232 if 'fa-youtube-play' not in li:
233 continue
234 lecture_url = self._search_regex(
235 r'<a[^>]+href=(["\'])(?P<url>(?:(?!\1).)+)\1', li,
236 'lecture url', default=None, group='url')
237 if not lecture_url:
238 continue
239 lecture_id = self._search_regex(
240 r'/lectures/(\d+)', lecture_url, 'lecture id', default=None)
241 title = self._html_search_regex(
242 r'<span[^>]+class=["\']lecture-name[^>]+>([^<]+)', li,
243 'title', default=None)
244 entry_url = urljoin(url_base, lecture_url)
245 if prefixed:
246 entry_url = self._URL_PREFIX + entry_url
247 entries.append(
248 self.url_result(
249 entry_url,
250 ie=TeachableIE.ie_key(), video_id=lecture_id,
251 video_title=clean_html(title)))
252
253 course_title = self._html_search_regex(
254 (r'(?s)<img[^>]+class=["\']course-image[^>]+>\s*<h\d>(.+?)</h',
255 r'(?s)<h\d[^>]+class=["\']course-title[^>]+>(.+?)</h'),
256 webpage, 'course title', fatal=False)
257
258 return self.playlist_result(entries, course_id, course_title)