Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/safari.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import json
   5 import re
   6
   7 from .common import InfoExtractor
   8
   9 from ..compat import (
  10     compat_parse_qs,
  11     compat_str,
  12     compat_urlparse,
  13 )
  14 from ..utils import (
  15     ExtractorError,
  16     update_url_query,
  17 )
  18
  19
  20 class SafariBaseIE(InfoExtractor):
  21     _LOGIN_URL = 'https://learning.oreilly.com/accounts/login/'
  22     _NETRC_MACHINE = 'safari'
  23
  24     _API_BASE = 'https://learning.oreilly.com/api/v1'
  25     _API_FORMAT = 'json'
  26
  27     LOGGED_IN = False
  28
  29     def _real_initialize(self):
  30         self._login()
  31
  32     def _login(self):
  33         username, password = self._get_login_info()
  34         if username is None:
  35             return
  36
  37         _, urlh = self._download_webpage_handle(
  38             'https://learning.oreilly.com/accounts/login-check/', None,
  39             'Downloading login page')
  40
  41         def is_logged(urlh):
  42             return 'learning.oreilly.com/home/' in compat_str(urlh.geturl())
  43
  44         if is_logged(urlh):
  45             self.LOGGED_IN = True
  46             return
  47
  48         redirect_url = compat_str(urlh.geturl())
  49         parsed_url = compat_urlparse.urlparse(redirect_url)
  50         qs = compat_parse_qs(parsed_url.query)
  51         next_uri = compat_urlparse.urljoin(
  52             'https://api.oreilly.com', qs['next'][0])
  53
  54         auth, urlh = self._download_json_handle(
  55             'https://www.oreilly.com/member/auth/login/', None, 'Logging in',
  56             data=json.dumps({
  57                 'email': username,
  58                 'password': password,
  59                 'redirect_uri': next_uri,
  60             }).encode(), headers={
  61                 'Content-Type': 'application/json',
  62                 'Referer': redirect_url,
  63             }, expected_status=400)
  64
  65         credentials = auth.get('credentials')
  66         if (not auth.get('logged_in') and not auth.get('redirect_uri')
  67                 and credentials):
  68             raise ExtractorError(
  69                 'Unable to login: %s' % credentials, expected=True)
  70
  71         # oreilly serves two same instances of the following cookies
  72         # in Set-Cookie header and expects first one to be actually set
  73         for cookie in ('groot_sessionid', 'orm-jwt', 'orm-rt'):
  74             self._apply_first_set_cookie_header(urlh, cookie)
  75
  76         _, urlh = self._download_webpage_handle(
  77             auth.get('redirect_uri') or next_uri, None, 'Completing login',)
  78
  79         if is_logged(urlh):
  80             self.LOGGED_IN = True
  81             return
  82
  83         raise ExtractorError('Unable to log in')
  84
  85
  86 class SafariIE(SafariBaseIE):
  87     IE_NAME = 'safari'
  88     IE_DESC = 'safaribooksonline.com online video'
  89     _VALID_URL = r'''(?x)
  90                         https?://
  91                             (?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/
  92                             (?:
  93                                 library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?\#&]+)\.html|
  94                                 videos/[^/]+/[^/]+/(?P<reference_id>[^-]+-[^/?\#&]+)
  95                             )
  96                     '''
  97
  98     _TESTS = [{
  99         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
 100         'md5': 'dcc5a425e79f2564148652616af1f2a3',
 101         'info_dict': {
 102             'id': '0_qbqx90ic',
 103             'ext': 'mp4',
 104             'title': 'Introduction to Hadoop Fundamentals LiveLessons',
 105             'timestamp': 1437758058,
 106             'upload_date': '20150724',
 107             'uploader_id': 'stork',
 108         },
 109     }, {
 110         # non-digits in course id
 111         'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
 112         'only_matching': True,
 113     }, {
 114         'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
 115         'only_matching': True,
 116     }, {
 117         'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314/9780134217314-PYMC_13_00',
 118         'only_matching': True,
 119     }, {
 120         'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838/9780133392838-00_SeriesIntro',
 121         'only_matching': True,
 122     }, {
 123         'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/00_SeriesIntro.html',
 124         'only_matching': True,
 125     }]
 126
 127     _PARTNER_ID = '1926081'
 128     _UICONF_ID = '29375172'
 129
 130     def _real_extract(self, url):
 131         mobj = re.match(self._VALID_URL, url)
 132
 133         reference_id = mobj.group('reference_id')
 134         if reference_id:
 135             video_id = reference_id
 136             partner_id = self._PARTNER_ID
 137             ui_id = self._UICONF_ID
 138         else:
 139             video_id = '%s-%s' % (mobj.group('course_id'), mobj.group('part'))
 140
 141             webpage, urlh = self._download_webpage_handle(url, video_id)
 142
 143             mobj = re.match(self._VALID_URL, urlh.geturl())
 144             reference_id = mobj.group('reference_id')
 145             if not reference_id:
 146                 reference_id = self._search_regex(
 147                     r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
 148                     webpage, 'kaltura reference id', group='id')
 149             partner_id = self._search_regex(
 150                 r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
 151                 webpage, 'kaltura widget id', default=self._PARTNER_ID,
 152                 group='id')
 153             ui_id = self._search_regex(
 154                 r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
 155                 webpage, 'kaltura uiconf id', default=self._UICONF_ID,
 156                 group='id')
 157
 158         query = {
 159             'wid': '_%s' % partner_id,
 160             'uiconf_id': ui_id,
 161             'flashvars[referenceId]': reference_id,
 162         }
 163
 164         if self.LOGGED_IN:
 165             kaltura_session = self._download_json(
 166                 '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
 167                 video_id, 'Downloading kaltura session JSON',
 168                 'Unable to download kaltura session JSON', fatal=False,
 169                 headers={'Accept': 'application/json'})
 170             if kaltura_session:
 171                 session = kaltura_session.get('session')
 172                 if session:
 173                     query['flashvars[ks]'] = session
 174
 175         return self.url_result(update_url_query(
 176             'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
 177             'Kaltura')
 178
 179
 180 class SafariApiIE(SafariBaseIE):
 181     IE_NAME = 'safari:api'
 182     _VALID_URL = r'https?://(?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
 183
 184     _TESTS = [{
 185         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
 186         'only_matching': True,
 187     }, {
 188         'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
 189         'only_matching': True,
 190     }]
 191
 192     def _real_extract(self, url):
 193         mobj = re.match(self._VALID_URL, url)
 194         part = self._download_json(
 195             url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
 196             'Downloading part JSON')
 197         return self.url_result(part['web_url'], SafariIE.ie_key())
 198
 199
 200 class SafariCourseIE(SafariBaseIE):
 201     IE_NAME = 'safari:course'
 202     IE_DESC = 'safaribooksonline.com online courses'
 203
 204     _VALID_URL = r'''(?x)
 205                     https?://
 206                         (?:
 207                             (?:www\.)?(?:safaribooksonline|(?:learning\.)?oreilly)\.com/
 208                             (?:
 209                                 library/view/[^/]+|
 210                                 api/v1/book|
 211                                 videos/[^/]+
 212                             )|
 213                             techbus\.safaribooksonline\.com
 214                         )
 215                         /(?P<id>[^/]+)
 216                     '''
 217
 218     _TESTS = [{
 219         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
 220         'info_dict': {
 221             'id': '9780133392838',
 222             'title': 'Hadoop Fundamentals LiveLessons',
 223         },
 224         'playlist_count': 22,
 225         'skip': 'Requires safaribooksonline account credentials',
 226     }, {
 227         'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
 228         'only_matching': True,
 229     }, {
 230         'url': 'http://techbus.safaribooksonline.com/9780134426365',
 231         'only_matching': True,
 232     }, {
 233         'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314',
 234         'only_matching': True,
 235     }, {
 236         'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838',
 237         'only_matching': True,
 238     }, {
 239         'url': 'https://www.oreilly.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
 240         'only_matching': True,
 241     }]
 242
 243     @classmethod
 244     def suitable(cls, url):
 245         return (False if SafariIE.suitable(url) or SafariApiIE.suitable(url)
 246                 else super(SafariCourseIE, cls).suitable(url))
 247
 248     def _real_extract(self, url):
 249         course_id = self._match_id(url)
 250
 251         course_json = self._download_json(
 252             '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
 253             course_id, 'Downloading course JSON')
 254
 255         if 'chapters' not in course_json:
 256             raise ExtractorError(
 257                 'No chapters found for course %s' % course_id, expected=True)
 258
 259         entries = [
 260             self.url_result(chapter, SafariApiIE.ie_key())
 261             for chapter in course_json['chapters']]
 262
 263         course_title = course_json['title']
 264
 265         return self.playlist_result(entries, course_id, course_title)