Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/safari.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7
   8 from ..utils import (
   9     ExtractorError,
  10     sanitized_Request,
  11     std_headers,
  12     urlencode_postdata,
  13     update_url_query,
  14 )
  15
  16
  17 class SafariBaseIE(InfoExtractor):
  18     _LOGIN_URL = 'https://www.safaribooksonline.com/accounts/login/'
  19     _NETRC_MACHINE = 'safari'
  20
  21     _API_BASE = 'https://www.safaribooksonline.com/api/v1'
  22     _API_FORMAT = 'json'
  23
  24     LOGGED_IN = False
  25
  26     def _real_initialize(self):
  27         self._login()
  28
  29     def _login(self):
  30         (username, password) = self._get_login_info()
  31         if username is None:
  32             return
  33
  34         headers = std_headers.copy()
  35         if 'Referer' not in headers:
  36             headers['Referer'] = self._LOGIN_URL
  37
  38         login_page = self._download_webpage(
  39             self._LOGIN_URL, None, 'Downloading login form', headers=headers)
  40
  41         def is_logged(webpage):
  42             return any(re.search(p, webpage) for p in (
  43                 r'href=["\']/accounts/logout/', r'>Sign Out<'))
  44
  45         if is_logged(login_page):
  46             self.LOGGED_IN = True
  47             return
  48
  49         csrf = self._html_search_regex(
  50             r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
  51             login_page, 'csrf token')
  52
  53         login_form = {
  54             'csrfmiddlewaretoken': csrf,
  55             'email': username,
  56             'password1': password,
  57             'login': 'Sign In',
  58             'next': '',
  59         }
  60
  61         request = sanitized_Request(
  62             self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
  63         login_page = self._download_webpage(
  64             request, None, 'Logging in as %s' % username)
  65
  66         if not is_logged(login_page):
  67             raise ExtractorError(
  68                 'Login failed; make sure your credentials are correct and try again.',
  69                 expected=True)
  70
  71         self.LOGGED_IN = True
  72
  73
  74 class SafariIE(SafariBaseIE):
  75     IE_NAME = 'safari'
  76     IE_DESC = 'safaribooksonline.com online video'
  77     _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?#&]+)\.html'
  78
  79     _TESTS = [{
  80         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
  81         'md5': 'dcc5a425e79f2564148652616af1f2a3',
  82         'info_dict': {
  83             'id': '0_qbqx90ic',
  84             'ext': 'mp4',
  85             'title': 'Introduction to Hadoop Fundamentals LiveLessons',
  86             'timestamp': 1437758058,
  87             'upload_date': '20150724',
  88             'uploader_id': 'stork',
  89         },
  90     }, {
  91         # non-digits in course id
  92         'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
  93         'only_matching': True,
  94     }, {
  95         'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
  96         'only_matching': True,
  97     }]
  98
  99     def _real_extract(self, url):
 100         mobj = re.match(self._VALID_URL, url)
 101         video_id = '%s/%s' % (mobj.group('course_id'), mobj.group('part'))
 102
 103         webpage = self._download_webpage(url, video_id)
 104         reference_id = self._search_regex(
 105             r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
 106             webpage, 'kaltura reference id', group='id')
 107         partner_id = self._search_regex(
 108             r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
 109             webpage, 'kaltura widget id', group='id')
 110         ui_id = self._search_regex(
 111             r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
 112             webpage, 'kaltura uiconf id', group='id')
 113
 114         query = {
 115             'wid': '_%s' % partner_id,
 116             'uiconf_id': ui_id,
 117             'flashvars[referenceId]': reference_id,
 118         }
 119
 120         if self.LOGGED_IN:
 121             kaltura_session = self._download_json(
 122                 '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
 123                 video_id, 'Downloading kaltura session JSON',
 124                 'Unable to download kaltura session JSON', fatal=False)
 125             if kaltura_session:
 126                 session = kaltura_session.get('session')
 127                 if session:
 128                     query['flashvars[ks]'] = session
 129
 130         return self.url_result(update_url_query(
 131             'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
 132             'Kaltura')
 133
 134
 135 class SafariApiIE(SafariBaseIE):
 136     IE_NAME = 'safari:api'
 137     _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
 138
 139     _TESTS = [{
 140         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
 141         'only_matching': True,
 142     }, {
 143         'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
 144         'only_matching': True,
 145     }]
 146
 147     def _real_extract(self, url):
 148         mobj = re.match(self._VALID_URL, url)
 149         part = self._download_json(
 150             url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
 151             'Downloading part JSON')
 152         return self.url_result(part['web_url'], SafariIE.ie_key())
 153
 154
 155 class SafariCourseIE(SafariBaseIE):
 156     IE_NAME = 'safari:course'
 157     IE_DESC = 'safaribooksonline.com online courses'
 158
 159     _VALID_URL = r'''(?x)
 160                     https?://
 161                         (?:
 162                             (?:www\.)?safaribooksonline\.com/(?:library/view/[^/]+|api/v1/book)|
 163                             techbus\.safaribooksonline\.com
 164                         )
 165                         /(?P<id>[^/]+)/?(?:[#?]|$)
 166                     '''
 167
 168     _TESTS = [{
 169         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
 170         'info_dict': {
 171             'id': '9780133392838',
 172             'title': 'Hadoop Fundamentals LiveLessons',
 173         },
 174         'playlist_count': 22,
 175         'skip': 'Requires safaribooksonline account credentials',
 176     }, {
 177         'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
 178         'only_matching': True,
 179     }, {
 180         'url': 'http://techbus.safaribooksonline.com/9780134426365',
 181         'only_matching': True,
 182     }]
 183
 184     def _real_extract(self, url):
 185         course_id = self._match_id(url)
 186
 187         course_json = self._download_json(
 188             '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
 189             course_id, 'Downloading course JSON')
 190
 191         if 'chapters' not in course_json:
 192             raise ExtractorError(
 193                 'No chapters found for course %s' % course_id, expected=True)
 194
 195         entries = [
 196             self.url_result(chapter, SafariApiIE.ie_key())
 197             for chapter in course_json['chapters']]
 198
 199         course_title = course_json['title']
 200
 201         return self.playlist_result(entries, course_id, course_title)