Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/packtpub.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import (
   8     compat_str,
   9     compat_HTTPError,
  10 )
  11 from ..utils import (
  12     clean_html,
  13     ExtractorError,
  14     remove_end,
  15     strip_or_none,
  16     unified_timestamp,
  17     urljoin,
  18 )
  19
  20
  21 class PacktPubBaseIE(InfoExtractor):
  22     _PACKT_BASE = 'https://www.packtpub.com'
  23     _MAPT_REST = '%s/mapt-rest' % _PACKT_BASE
  24
  25
  26 class PacktPubIE(PacktPubBaseIE):
  27     _VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>\d+)/(?P<id>\d+)'
  28
  29     _TESTS = [{
  30         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
  31         'md5': '1e74bd6cfd45d7d07666f4684ef58f70',
  32         'info_dict': {
  33             'id': '20530',
  34             'ext': 'mp4',
  35             'title': 'Project Intro',
  36             'thumbnail': r're:(?i)^https?://.*\.jpg',
  37             'timestamp': 1490918400,
  38             'upload_date': '20170331',
  39         },
  40     }, {
  41         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215/20528/20530/project-intro',
  42         'only_matching': True,
  43     }]
  44     _NETRC_MACHINE = 'packtpub'
  45     _TOKEN = None
  46
  47     def _real_initialize(self):
  48         username, password = self._get_login_info()
  49         if username is None:
  50             return
  51         try:
  52             self._TOKEN = self._download_json(
  53                 self._MAPT_REST + '/users/tokens', None,
  54                 'Downloading Authorization Token', data=json.dumps({
  55                     'email': username,
  56                     'password': password,
  57                 }).encode())['data']['access']
  58         except ExtractorError as e:
  59             if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 404):
  60                 message = self._parse_json(e.cause.read().decode(), None)['message']
  61                 raise ExtractorError(message, expected=True)
  62             raise
  63
  64     def _handle_error(self, response):
  65         if response.get('status') != 'success':
  66             raise ExtractorError(
  67                 '% said: %s' % (self.IE_NAME, response['message']),
  68                 expected=True)
  69
  70     def _download_json(self, *args, **kwargs):
  71         response = super(PacktPubIE, self)._download_json(*args, **kwargs)
  72         self._handle_error(response)
  73         return response
  74
  75     def _real_extract(self, url):
  76         mobj = re.match(self._VALID_URL, url)
  77         course_id, chapter_id, video_id = mobj.group(
  78             'course_id', 'chapter_id', 'id')
  79
  80         headers = {}
  81         if self._TOKEN:
  82             headers['Authorization'] = 'Bearer ' + self._TOKEN
  83         video = self._download_json(
  84             '%s/users/me/products/%s/chapters/%s/sections/%s'
  85             % (self._MAPT_REST, course_id, chapter_id, video_id), video_id,
  86             'Downloading JSON video', headers=headers)['data']
  87
  88         content = video.get('content')
  89         if not content:
  90             self.raise_login_required('This video is locked')
  91
  92         video_url = content['file']
  93
  94         metadata = self._download_json(
  95             '%s/products/%s/chapters/%s/sections/%s/metadata'
  96             % (self._MAPT_REST, course_id, chapter_id, video_id),
  97             video_id)['data']
  98
  99         title = metadata['pageTitle']
 100         course_title = metadata.get('title')
 101         if course_title:
 102             title = remove_end(title, ' - %s' % course_title)
 103         timestamp = unified_timestamp(metadata.get('publicationDate'))
 104         thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
 105
 106         return {
 107             'id': video_id,
 108             'url': video_url,
 109             'title': title,
 110             'thumbnail': thumbnail,
 111             'timestamp': timestamp,
 112         }
 113
 114
 115 class PacktPubCourseIE(PacktPubBaseIE):
 116     _VALID_URL = r'(?P<url>https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<id>\d+))'
 117     _TESTS = [{
 118         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215',
 119         'info_dict': {
 120             'id': '9781787122215',
 121             'title': 'Learn Nodejs by building 12 projects [Video]',
 122         },
 123         'playlist_count': 90,
 124     }, {
 125         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215',
 126         'only_matching': True,
 127     }]
 128
 129     @classmethod
 130     def suitable(cls, url):
 131         return False if PacktPubIE.suitable(url) else super(
 132             PacktPubCourseIE, cls).suitable(url)
 133
 134     def _real_extract(self, url):
 135         mobj = re.match(self._VALID_URL, url)
 136         url, course_id = mobj.group('url', 'id')
 137
 138         course = self._download_json(
 139             '%s/products/%s/metadata' % (self._MAPT_REST, course_id),
 140             course_id)['data']
 141
 142         entries = []
 143         for chapter_num, chapter in enumerate(course['tableOfContents'], 1):
 144             if chapter.get('type') != 'chapter':
 145                 continue
 146             children = chapter.get('children')
 147             if not isinstance(children, list):
 148                 continue
 149             chapter_info = {
 150                 'chapter': chapter.get('title'),
 151                 'chapter_number': chapter_num,
 152                 'chapter_id': chapter.get('id'),
 153             }
 154             for section in children:
 155                 if section.get('type') != 'section':
 156                     continue
 157                 section_url = section.get('seoUrl')
 158                 if not isinstance(section_url, compat_str):
 159                     continue
 160                 entry = {
 161                     '_type': 'url_transparent',
 162                     'url': urljoin(url + '/', section_url),
 163                     'title': strip_or_none(section.get('title')),
 164                     'description': clean_html(section.get('summary')),
 165                     'ie_key': PacktPubIE.ie_key(),
 166                 }
 167                 entry.update(chapter_info)
 168                 entries.append(entry)
 169
 170         return self.playlist_result(entries, course_id, course.get('title'))