Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/packtpub.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import (
   8     # compat_str,
   9     compat_HTTPError,
  10 )
  11 from ..utils import (
  12     clean_html,
  13     ExtractorError,
  14     # remove_end,
  15     str_or_none,
  16     strip_or_none,
  17     unified_timestamp,
  18     # urljoin,
  19 )
  20
  21
  22 class PacktPubBaseIE(InfoExtractor):
  23     # _PACKT_BASE = 'https://www.packtpub.com'
  24     _STATIC_PRODUCTS_BASE = 'https://static.packt-cdn.com/products/'
  25
  26
  27 class PacktPubIE(PacktPubBaseIE):
  28     _VALID_URL = r'https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<course_id>\d+)/(?P<chapter_id>[^/]+)/(?P<id>[^/]+)(?:/(?P<display_id>[^/?&#]+))?'
  29
  30     _TESTS = [{
  31         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215/20528/20530/Project+Intro',
  32         'md5': '1e74bd6cfd45d7d07666f4684ef58f70',
  33         'info_dict': {
  34             'id': '20530',
  35             'ext': 'mp4',
  36             'title': 'Project Intro',
  37             'thumbnail': r're:(?i)^https?://.*\.jpg',
  38             'timestamp': 1490918400,
  39             'upload_date': '20170331',
  40         },
  41     }, {
  42         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215/20528/20530/project-intro',
  43         'only_matching': True,
  44     }, {
  45         'url': 'https://subscription.packtpub.com/video/programming/9781838988906/p1/video1_1/business-card-project',
  46         'only_matching': True,
  47     }]
  48     _NETRC_MACHINE = 'packtpub'
  49     _TOKEN = None
  50
  51     def _real_initialize(self):
  52         username, password = self._get_login_info()
  53         if username is None:
  54             return
  55         try:
  56             self._TOKEN = self._download_json(
  57                 'https://services.packtpub.com/auth-v1/users/tokens', None,
  58                 'Downloading Authorization Token', data=json.dumps({
  59                     'username': username,
  60                     'password': password,
  61                 }).encode())['data']['access']
  62         except ExtractorError as e:
  63             if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 404):
  64                 message = self._parse_json(e.cause.read().decode(), None)['message']
  65                 raise ExtractorError(message, expected=True)
  66             raise
  67
  68     def _real_extract(self, url):
  69         course_id, chapter_id, video_id, display_id = re.match(self._VALID_URL, url).groups()
  70
  71         headers = {}
  72         if self._TOKEN:
  73             headers['Authorization'] = 'Bearer ' + self._TOKEN
  74         try:
  75             video_url = self._download_json(
  76                 'https://services.packtpub.com/products-v1/products/%s/%s/%s' % (course_id, chapter_id, video_id), video_id,
  77                 'Downloading JSON video', headers=headers)['data']
  78         except ExtractorError as e:
  79             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
  80                 self.raise_login_required('This video is locked')
  81             raise
  82
  83         # TODO: find a better way to avoid duplicating course requests
  84         # metadata = self._download_json(
  85         #     '%s/products/%s/chapters/%s/sections/%s/metadata'
  86         #     % (self._MAPT_REST, course_id, chapter_id, video_id),
  87         #     video_id)['data']
  88
  89         # title = metadata['pageTitle']
  90         # course_title = metadata.get('title')
  91         # if course_title:
  92         #     title = remove_end(title, ' - %s' % course_title)
  93         # timestamp = unified_timestamp(metadata.get('publicationDate'))
  94         # thumbnail = urljoin(self._PACKT_BASE, metadata.get('filepath'))
  95
  96         return {
  97             'id': video_id,
  98             'url': video_url,
  99             'title': display_id or video_id,  # title,
 100             # 'thumbnail': thumbnail,
 101             # 'timestamp': timestamp,
 102         }
 103
 104
 105 class PacktPubCourseIE(PacktPubBaseIE):
 106     _VALID_URL = r'(?P<url>https?://(?:(?:www\.)?packtpub\.com/mapt|subscription\.packtpub\.com)/video/[^/]+/(?P<id>\d+))'
 107     _TESTS = [{
 108         'url': 'https://www.packtpub.com/mapt/video/web-development/9781787122215',
 109         'info_dict': {
 110             'id': '9781787122215',
 111             'title': 'Learn Nodejs by building 12 projects [Video]',
 112             'description': 'md5:489da8d953f416e51927b60a1c7db0aa',
 113         },
 114         'playlist_count': 90,
 115     }, {
 116         'url': 'https://subscription.packtpub.com/video/web_development/9781787122215',
 117         'only_matching': True,
 118     }]
 119
 120     @classmethod
 121     def suitable(cls, url):
 122         return False if PacktPubIE.suitable(url) else super(
 123             PacktPubCourseIE, cls).suitable(url)
 124
 125     def _real_extract(self, url):
 126         mobj = re.match(self._VALID_URL, url)
 127         url, course_id = mobj.group('url', 'id')
 128
 129         course = self._download_json(
 130             self._STATIC_PRODUCTS_BASE + '%s/toc' % course_id, course_id)
 131         metadata = self._download_json(
 132             self._STATIC_PRODUCTS_BASE + '%s/summary' % course_id,
 133             course_id, fatal=False) or {}
 134
 135         entries = []
 136         for chapter_num, chapter in enumerate(course['chapters'], 1):
 137             chapter_id = str_or_none(chapter.get('id'))
 138             sections = chapter.get('sections')
 139             if not chapter_id or not isinstance(sections, list):
 140                 continue
 141             chapter_info = {
 142                 'chapter': chapter.get('title'),
 143                 'chapter_number': chapter_num,
 144                 'chapter_id': chapter_id,
 145             }
 146             for section in sections:
 147                 section_id = str_or_none(section.get('id'))
 148                 if not section_id or section.get('contentType') != 'video':
 149                     continue
 150                 entry = {
 151                     '_type': 'url_transparent',
 152                     'url': '/'.join([url, chapter_id, section_id]),
 153                     'title': strip_or_none(section.get('title')),
 154                     'description': clean_html(section.get('summary')),
 155                     'thumbnail': metadata.get('coverImage'),
 156                     'timestamp': unified_timestamp(metadata.get('publicationDate')),
 157                     'ie_key': PacktPubIE.ie_key(),
 158                 }
 159                 entry.update(chapter_info)
 160                 entries.append(entry)
 161
 162         return self.playlist_result(
 163             entries, course_id, metadata.get('title'),
 164             clean_html(metadata.get('about')))