]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/safari.py
Update changelog.
[youtubedl] / youtube_dl / extractor / safari.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7
8 from ..utils import (
9 ExtractorError,
10 sanitized_Request,
11 std_headers,
12 urlencode_postdata,
13 update_url_query,
14 )
15
16
17 class SafariBaseIE(InfoExtractor):
18 _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
19 _NETRC_MACHINE = 'safari'
20
21 _API_BASE = 'https://www.safaribooksonline.com/api/v1'
22 _API_FORMAT = 'json'
23
24 LOGGED_IN = False
25
26 def _real_initialize(self):
27 self._login()
28
29 def _login(self):
30 username, password = self._get_login_info()
31 if username is None:
32 return
33
34 headers = std_headers.copy()
35 if 'Referer' not in headers:
36 headers['Referer'] = self._LOGIN_URL
37
38 login_page = self._download_webpage(
39 self._LOGIN_URL, None, 'Downloading login form', headers=headers)
40
41 def is_logged(webpage):
42 return any(re.search(p, webpage) for p in (
43 r'href=["\']/accounts/logout/', r'>Sign Out<'))
44
45 if is_logged(login_page):
46 self.LOGGED_IN = True
47 return
48
49 csrf = self._html_search_regex(
50 r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
51 login_page, 'csrf token')
52
53 login_form = {
54 'csrfmiddlewaretoken': csrf,
55 'email': username,
56 'password1': password,
57 'login': 'Sign In',
58 'next': '',
59 }
60
61 request = sanitized_Request(
62 self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
63 login_page = self._download_webpage(
64 request, None, 'Logging in')
65
66 if not is_logged(login_page):
67 raise ExtractorError(
68 'Login failed; make sure your credentials are correct and try again.',
69 expected=True)
70
71 self.LOGGED_IN = True
72
73
74 class SafariIE(SafariBaseIE):
75 IE_NAME = 'safari'
76 IE_DESC = 'safaribooksonline.com online video'
77 _VALID_URL = r'''(?x)
78 https?://
79 (?:www\.)?safaribooksonline\.com/
80 (?:
81 library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?\#&]+)\.html|
82 videos/[^/]+/[^/]+/(?P<reference_id>[^-]+-[^/?\#&]+)
83 )
84 '''
85
86 _TESTS = [{
87 'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
88 'md5': 'dcc5a425e79f2564148652616af1f2a3',
89 'info_dict': {
90 'id': '0_qbqx90ic',
91 'ext': 'mp4',
92 'title': 'Introduction to Hadoop Fundamentals LiveLessons',
93 'timestamp': 1437758058,
94 'upload_date': '20150724',
95 'uploader_id': 'stork',
96 },
97 }, {
98 # non-digits in course id
99 'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
100 'only_matching': True,
101 }, {
102 'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
103 'only_matching': True,
104 }, {
105 'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314/9780134217314-PYMC_13_00',
106 'only_matching': True,
107 }]
108
109 _PARTNER_ID = '1926081'
110 _UICONF_ID = '29375172'
111
112 def _real_extract(self, url):
113 mobj = re.match(self._VALID_URL, url)
114
115 reference_id = mobj.group('reference_id')
116 if reference_id:
117 video_id = reference_id
118 partner_id = self._PARTNER_ID
119 ui_id = self._UICONF_ID
120 else:
121 video_id = '%s-%s' % (mobj.group('course_id'), mobj.group('part'))
122
123 webpage, urlh = self._download_webpage_handle(url, video_id)
124
125 mobj = re.match(self._VALID_URL, urlh.geturl())
126 reference_id = mobj.group('reference_id')
127 if not reference_id:
128 reference_id = self._search_regex(
129 r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
130 webpage, 'kaltura reference id', group='id')
131 partner_id = self._search_regex(
132 r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
133 webpage, 'kaltura widget id', default=self._PARTNER_ID,
134 group='id')
135 ui_id = self._search_regex(
136 r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
137 webpage, 'kaltura uiconf id', default=self._UICONF_ID,
138 group='id')
139
140 query = {
141 'wid': '_%s' % partner_id,
142 'uiconf_id': ui_id,
143 'flashvars[referenceId]': reference_id,
144 }
145
146 if self.LOGGED_IN:
147 kaltura_session = self._download_json(
148 '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
149 video_id, 'Downloading kaltura session JSON',
150 'Unable to download kaltura session JSON', fatal=False)
151 if kaltura_session:
152 session = kaltura_session.get('session')
153 if session:
154 query['flashvars[ks]'] = session
155
156 return self.url_result(update_url_query(
157 'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
158 'Kaltura')
159
160
161 class SafariApiIE(SafariBaseIE):
162 IE_NAME = 'safari:api'
163 _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
164
165 _TESTS = [{
166 'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
167 'only_matching': True,
168 }, {
169 'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
170 'only_matching': True,
171 }]
172
173 def _real_extract(self, url):
174 mobj = re.match(self._VALID_URL, url)
175 part = self._download_json(
176 url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
177 'Downloading part JSON')
178 return self.url_result(part['web_url'], SafariIE.ie_key())
179
180
181 class SafariCourseIE(SafariBaseIE):
182 IE_NAME = 'safari:course'
183 IE_DESC = 'safaribooksonline.com online courses'
184
185 _VALID_URL = r'''(?x)
186 https?://
187 (?:
188 (?:www\.)?safaribooksonline\.com/
189 (?:
190 library/view/[^/]+|
191 api/v1/book|
192 videos/[^/]+
193 )|
194 techbus\.safaribooksonline\.com
195 )
196 /(?P<id>[^/]+)
197 '''
198
199 _TESTS = [{
200 'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
201 'info_dict': {
202 'id': '9780133392838',
203 'title': 'Hadoop Fundamentals LiveLessons',
204 },
205 'playlist_count': 22,
206 'skip': 'Requires safaribooksonline account credentials',
207 }, {
208 'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
209 'only_matching': True,
210 }, {
211 'url': 'http://techbus.safaribooksonline.com/9780134426365',
212 'only_matching': True,
213 }, {
214 'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314',
215 'only_matching': True,
216 }]
217
218 @classmethod
219 def suitable(cls, url):
220 return (False if SafariIE.suitable(url) or SafariApiIE.suitable(url)
221 else super(SafariCourseIE, cls).suitable(url))
222
223 def _real_extract(self, url):
224 course_id = self._match_id(url)
225
226 course_json = self._download_json(
227 '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
228 course_id, 'Downloading course JSON')
229
230 if 'chapters' not in course_json:
231 raise ExtractorError(
232 'No chapters found for course %s' % course_id, expected=True)
233
234 entries = [
235 self.url_result(chapter, SafariApiIE.ie_key())
236 for chapter in course_json['chapters']]
237
238 course_title = course_json['title']
239
240 return self.playlist_result(entries, course_id, course_title)