]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/safari.py
Fix extraction from youtube.
[youtubedl] / youtube_dl / extractor / safari.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7
8 from ..utils import (
9 ExtractorError,
10 sanitized_Request,
11 std_headers,
12 urlencode_postdata,
13 update_url_query,
14 )
15
16
17 class SafariBaseIE(InfoExtractor):
18 _LOGIN_URL = 'https://learning.oreilly.com/accounts/login/'
19 _NETRC_MACHINE = 'safari'
20
21 _API_BASE = 'https://learning.oreilly.com/api/v1'
22 _API_FORMAT = 'json'
23
24 LOGGED_IN = False
25
26 def _real_initialize(self):
27 self._login()
28
29 def _login(self):
30 username, password = self._get_login_info()
31 if username is None:
32 return
33
34 headers = std_headers.copy()
35 if 'Referer' not in headers:
36 headers['Referer'] = self._LOGIN_URL
37
38 login_page = self._download_webpage(
39 self._LOGIN_URL, None, 'Downloading login form', headers=headers)
40
41 def is_logged(webpage):
42 return any(re.search(p, webpage) for p in (
43 r'href=["\']/accounts/logout/', r'>Sign Out<'))
44
45 if is_logged(login_page):
46 self.LOGGED_IN = True
47 return
48
49 csrf = self._html_search_regex(
50 r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
51 login_page, 'csrf token')
52
53 login_form = {
54 'csrfmiddlewaretoken': csrf,
55 'email': username,
56 'password1': password,
57 'login': 'Sign In',
58 'next': '',
59 }
60
61 request = sanitized_Request(
62 self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
63 login_page = self._download_webpage(
64 request, None, 'Logging in')
65
66 if not is_logged(login_page):
67 raise ExtractorError(
68 'Login failed; make sure your credentials are correct and try again.',
69 expected=True)
70
71 self.LOGGED_IN = True
72
73
74 class SafariIE(SafariBaseIE):
75 IE_NAME = 'safari'
76 IE_DESC = 'safaribooksonline.com online video'
77 _VALID_URL = r'''(?x)
78 https?://
79 (?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/
80 (?:
81 library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?\#&]+)\.html|
82 videos/[^/]+/[^/]+/(?P<reference_id>[^-]+-[^/?\#&]+)
83 )
84 '''
85
86 _TESTS = [{
87 'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
88 'md5': 'dcc5a425e79f2564148652616af1f2a3',
89 'info_dict': {
90 'id': '0_qbqx90ic',
91 'ext': 'mp4',
92 'title': 'Introduction to Hadoop Fundamentals LiveLessons',
93 'timestamp': 1437758058,
94 'upload_date': '20150724',
95 'uploader_id': 'stork',
96 },
97 }, {
98 # non-digits in course id
99 'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
100 'only_matching': True,
101 }, {
102 'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
103 'only_matching': True,
104 }, {
105 'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314/9780134217314-PYMC_13_00',
106 'only_matching': True,
107 }, {
108 'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838/9780133392838-00_SeriesIntro',
109 'only_matching': True,
110 }]
111
112 _PARTNER_ID = '1926081'
113 _UICONF_ID = '29375172'
114
115 def _real_extract(self, url):
116 mobj = re.match(self._VALID_URL, url)
117
118 reference_id = mobj.group('reference_id')
119 if reference_id:
120 video_id = reference_id
121 partner_id = self._PARTNER_ID
122 ui_id = self._UICONF_ID
123 else:
124 video_id = '%s-%s' % (mobj.group('course_id'), mobj.group('part'))
125
126 webpage, urlh = self._download_webpage_handle(url, video_id)
127
128 mobj = re.match(self._VALID_URL, urlh.geturl())
129 reference_id = mobj.group('reference_id')
130 if not reference_id:
131 reference_id = self._search_regex(
132 r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
133 webpage, 'kaltura reference id', group='id')
134 partner_id = self._search_regex(
135 r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
136 webpage, 'kaltura widget id', default=self._PARTNER_ID,
137 group='id')
138 ui_id = self._search_regex(
139 r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
140 webpage, 'kaltura uiconf id', default=self._UICONF_ID,
141 group='id')
142
143 query = {
144 'wid': '_%s' % partner_id,
145 'uiconf_id': ui_id,
146 'flashvars[referenceId]': reference_id,
147 }
148
149 if self.LOGGED_IN:
150 kaltura_session = self._download_json(
151 '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
152 video_id, 'Downloading kaltura session JSON',
153 'Unable to download kaltura session JSON', fatal=False)
154 if kaltura_session:
155 session = kaltura_session.get('session')
156 if session:
157 query['flashvars[ks]'] = session
158
159 return self.url_result(update_url_query(
160 'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
161 'Kaltura')
162
163
164 class SafariApiIE(SafariBaseIE):
165 IE_NAME = 'safari:api'
166 _VALID_URL = r'https?://(?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
167
168 _TESTS = [{
169 'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
170 'only_matching': True,
171 }, {
172 'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
173 'only_matching': True,
174 }]
175
176 def _real_extract(self, url):
177 mobj = re.match(self._VALID_URL, url)
178 part = self._download_json(
179 url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
180 'Downloading part JSON')
181 return self.url_result(part['web_url'], SafariIE.ie_key())
182
183
184 class SafariCourseIE(SafariBaseIE):
185 IE_NAME = 'safari:course'
186 IE_DESC = 'safaribooksonline.com online courses'
187
188 _VALID_URL = r'''(?x)
189 https?://
190 (?:
191 (?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/
192 (?:
193 library/view/[^/]+|
194 api/v1/book|
195 videos/[^/]+
196 )|
197 techbus\.safaribooksonline\.com
198 )
199 /(?P<id>[^/]+)
200 '''
201
202 _TESTS = [{
203 'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
204 'info_dict': {
205 'id': '9780133392838',
206 'title': 'Hadoop Fundamentals LiveLessons',
207 },
208 'playlist_count': 22,
209 'skip': 'Requires safaribooksonline account credentials',
210 }, {
211 'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
212 'only_matching': True,
213 }, {
214 'url': 'http://techbus.safaribooksonline.com/9780134426365',
215 'only_matching': True,
216 }, {
217 'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314',
218 'only_matching': True,
219 }, {
220 'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838',
221 'only_matching': True,
222 }]
223
224 @classmethod
225 def suitable(cls, url):
226 return (False if SafariIE.suitable(url) or SafariApiIE.suitable(url)
227 else super(SafariCourseIE, cls).suitable(url))
228
229 def _real_extract(self, url):
230 course_id = self._match_id(url)
231
232 course_json = self._download_json(
233 '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
234 course_id, 'Downloading course JSON')
235
236 if 'chapters' not in course_json:
237 raise ExtractorError(
238 'No chapters found for course %s' % course_id, expected=True)
239
240 entries = [
241 self.url_result(chapter, SafariApiIE.ie_key())
242 for chapter in course_json['chapters']]
243
244 course_title = course_json['title']
245
246 return self.playlist_result(entries, course_id, course_title)