Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/viewlift.py

   1 from __future__ import unicode_literals
   2
   3 import base64
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import compat_urllib_parse_unquote
   8 from ..utils import (
   9     ExtractorError,
  10     clean_html,
  11     determine_ext,
  12     int_or_none,
  13     js_to_json,
  14     parse_age_limit,
  15     parse_duration,
  16     try_get,
  17 )
  18
  19
  20 class ViewLiftBaseIE(InfoExtractor):
  21     _DOMAINS_REGEX = r'(?:(?:main\.)?snagfilms|snagxtreme|funnyforfree|kiddovid|winnersview|(?:monumental|lax)sportsnetwork|vayafilm)\.com|hoichoi\.tv'
  22
  23
  24 class ViewLiftEmbedIE(ViewLiftBaseIE):
  25     _VALID_URL = r'https?://(?:(?:www|embed)\.)?(?:%s)/embed/player\?.*\bfilmId=(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})' % ViewLiftBaseIE._DOMAINS_REGEX
  26     _TESTS = [{
  27         'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831&w=500',
  28         'md5': '2924e9215c6eff7a55ed35b72276bd93',
  29         'info_dict': {
  30             'id': '74849a00-85a9-11e1-9660-123139220831',
  31             'ext': 'mp4',
  32             'title': '#whilewewatch',
  33         }
  34     }, {
  35         # invalid labels, 360p is better that 480p
  36         'url': 'http://www.snagfilms.com/embed/player?filmId=17ca0950-a74a-11e0-a92a-0026bb61d036',
  37         'md5': '882fca19b9eb27ef865efeeaed376a48',
  38         'info_dict': {
  39             'id': '17ca0950-a74a-11e0-a92a-0026bb61d036',
  40             'ext': 'mp4',
  41             'title': 'Life in Limbo',
  42         }
  43     }, {
  44         'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
  45         'only_matching': True,
  46     }]
  47
  48     @staticmethod
  49     def _extract_url(webpage):
  50         mobj = re.search(
  51             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?(?:%s)/embed/player.+?)\1' % ViewLiftBaseIE._DOMAINS_REGEX,
  52             webpage)
  53         if mobj:
  54             return mobj.group('url')
  55
  56     def _real_extract(self, url):
  57         video_id = self._match_id(url)
  58
  59         webpage = self._download_webpage(url, video_id)
  60
  61         if '>This film is not playable in your area.<' in webpage:
  62             raise ExtractorError(
  63                 'Film %s is not playable in your area.' % video_id, expected=True)
  64
  65         formats = []
  66         has_bitrate = False
  67         sources = self._parse_json(self._search_regex(
  68             r'(?s)sources:\s*(\[.+?\]),', webpage,
  69             'sources', default='[]'), video_id, js_to_json)
  70         for source in sources:
  71             file_ = source.get('file')
  72             if not file_:
  73                 continue
  74             type_ = source.get('type')
  75             ext = determine_ext(file_)
  76             format_id = source.get('label') or ext
  77             if all(v in ('m3u8', 'hls') for v in (type_, ext)):
  78                 formats.extend(self._extract_m3u8_formats(
  79                     file_, video_id, 'mp4', 'm3u8_native',
  80                     m3u8_id='hls', fatal=False))
  81             else:
  82                 bitrate = int_or_none(self._search_regex(
  83                     [r'(\d+)kbps', r'_\d{1,2}x\d{1,2}_(\d{3,})\.%s' % ext],
  84                     file_, 'bitrate', default=None))
  85                 if not has_bitrate and bitrate:
  86                     has_bitrate = True
  87                 height = int_or_none(self._search_regex(
  88                     r'^(\d+)[pP]$', format_id, 'height', default=None))
  89                 formats.append({
  90                     'url': file_,
  91                     'format_id': 'http-%s%s' % (format_id, ('-%dk' % bitrate if bitrate else '')),
  92                     'tbr': bitrate,
  93                     'height': height,
  94                 })
  95         if not formats:
  96             hls_url = self._parse_json(self._search_regex(
  97                 r'filmInfo\.src\s*=\s*({.+?});',
  98                 webpage, 'src'), video_id, js_to_json)['src']
  99             formats = self._extract_m3u8_formats(
 100                 hls_url, video_id, 'mp4', 'm3u8_native',
 101                 m3u8_id='hls', fatal=False)
 102         field_preference = None if has_bitrate else ('height', 'tbr', 'format_id')
 103         self._sort_formats(formats, field_preference)
 104
 105         title = self._search_regex(
 106             [r"title\s*:\s*'([^']+)'", r'<title>([^<]+)</title>'],
 107             webpage, 'title')
 108
 109         return {
 110             'id': video_id,
 111             'title': title,
 112             'formats': formats,
 113         }
 114
 115
 116 class ViewLiftIE(ViewLiftBaseIE):
 117     _VALID_URL = r'https?://(?:www\.)?(?P<domain>%s)(?:/(?:films/title|show|(?:news/)?videos?))?/(?P<id>[^?#]+)' % ViewLiftBaseIE._DOMAINS_REGEX
 118     _TESTS = [{
 119         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
 120         'md5': '19844f897b35af219773fd63bdec2942',
 121         'info_dict': {
 122             'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
 123             'display_id': 'lost_for_life',
 124             'ext': 'mp4',
 125             'title': 'Lost for Life',
 126             'description': 'md5:ea10b5a50405ae1f7b5269a6ec594102',
 127             'thumbnail': r're:^https?://.*\.jpg',
 128             'duration': 4489,
 129             'categories': 'mincount:3',
 130             'age_limit': 14,
 131             'upload_date': '20150421',
 132             'timestamp': 1429656820,
 133         }
 134     }, {
 135         'url': 'http://www.snagfilms.com/show/the_world_cut_project/india',
 136         'md5': 'e6292e5b837642bbda82d7f8bf3fbdfd',
 137         'info_dict': {
 138             'id': '00000145-d75c-d96e-a9c7-ff5c67b20000',
 139             'display_id': 'the_world_cut_project/india',
 140             'ext': 'mp4',
 141             'title': 'India',
 142             'description': 'md5:5c168c5a8f4719c146aad2e0dfac6f5f',
 143             'thumbnail': r're:^https?://.*\.jpg',
 144             'duration': 979,
 145             'timestamp': 1399478279,
 146             'upload_date': '20140507',
 147         }
 148     }, {
 149         'url': 'http://main.snagfilms.com/augie_alone/s_2_ep_12_love',
 150         'info_dict': {
 151             'id': '00000148-7b53-de26-a9fb-fbf306f70020',
 152             'display_id': 'augie_alone/s_2_ep_12_love',
 153             'ext': 'mp4',
 154             'title': 'Augie, Alone:S. 2 Ep. 12 - Love',
 155             'description': 'md5:db2a5c72d994f16a780c1eb353a8f403',
 156             'thumbnail': r're:^https?://.*\.jpg',
 157             'duration': 107,
 158         },
 159         'params': {
 160             'skip_download': True,
 161         },
 162     }, {
 163         'url': 'http://main.snagfilms.com/films/title/the_freebie',
 164         'only_matching': True,
 165     }, {
 166         # Film is not playable in your area.
 167         'url': 'http://www.snagfilms.com/films/title/inside_mecca',
 168         'only_matching': True,
 169     }, {
 170         # Film is not available.
 171         'url': 'http://www.snagfilms.com/show/augie_alone/flirting',
 172         'only_matching': True,
 173     }, {
 174         'url': 'http://www.winnersview.com/videos/the-good-son',
 175         'only_matching': True,
 176     }, {
 177         # Was once Kaltura embed
 178         'url': 'https://www.monumentalsportsnetwork.com/videos/john-carlson-postgame-2-25-15',
 179         'only_matching': True,
 180     }]
 181
 182     @classmethod
 183     def suitable(cls, url):
 184         return False if ViewLiftEmbedIE.suitable(url) else super(ViewLiftIE, cls).suitable(url)
 185
 186     def _real_extract(self, url):
 187         domain, display_id = re.match(self._VALID_URL, url).groups()
 188
 189         webpage = self._download_webpage(url, display_id)
 190
 191         if ">Sorry, the Film you're looking for is not available.<" in webpage:
 192             raise ExtractorError(
 193                 'Film %s is not available.' % display_id, expected=True)
 194
 195         initial_store_state = self._search_regex(
 196             r"window\.initialStoreState\s*=.*?JSON\.parse\(unescape\(atob\('([^']+)'\)\)\)",
 197             webpage, 'Initial Store State', default=None)
 198         if initial_store_state:
 199             modules = self._parse_json(compat_urllib_parse_unquote(base64.b64decode(
 200                 initial_store_state).decode()), display_id)['page']['data']['modules']
 201             content_data = next(m['contentData'][0] for m in modules if m.get('moduleType') == 'VideoDetailModule')
 202             gist = content_data['gist']
 203             film_id = gist['id']
 204             title = gist['title']
 205             video_assets = try_get(
 206                 content_data, lambda x: x['streamingInfo']['videoAssets'], dict)
 207             if not video_assets:
 208                 token = self._download_json(
 209                     'https://prod-api.viewlift.com/identity/anonymous-token',
 210                     film_id, 'Downloading authorization token',
 211                     query={'site': 'snagfilms'})['authorizationToken']
 212                 video_assets = self._download_json(
 213                     'https://prod-api.viewlift.com/entitlement/video/status',
 214                     film_id, headers={
 215                         'Authorization': token,
 216                         'Referer': url,
 217                     }, query={
 218                         'id': film_id
 219                     })['video']['streamingInfo']['videoAssets']
 220
 221             formats = []
 222             mpeg_video_assets = video_assets.get('mpeg') or []
 223             for video_asset in mpeg_video_assets:
 224                 video_asset_url = video_asset.get('url')
 225                 if not video_asset:
 226                     continue
 227                 bitrate = int_or_none(video_asset.get('bitrate'))
 228                 height = int_or_none(self._search_regex(
 229                     r'^_?(\d+)[pP]$', video_asset.get('renditionValue'),
 230                     'height', default=None))
 231                 formats.append({
 232                     'url': video_asset_url,
 233                     'format_id': 'http%s' % ('-%d' % bitrate if bitrate else ''),
 234                     'tbr': bitrate,
 235                     'height': height,
 236                     'vcodec': video_asset.get('codec'),
 237                 })
 238
 239             hls_url = video_assets.get('hls')
 240             if hls_url:
 241                 formats.extend(self._extract_m3u8_formats(
 242                     hls_url, film_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 243             self._sort_formats(formats, ('height', 'tbr', 'format_id'))
 244
 245             info = {
 246                 'id': film_id,
 247                 'display_id': display_id,
 248                 'title': title,
 249                 'description': gist.get('description'),
 250                 'thumbnail': gist.get('videoImageUrl'),
 251                 'duration': int_or_none(gist.get('runtime')),
 252                 'age_limit': parse_age_limit(content_data.get('parentalRating')),
 253                 'timestamp': int_or_none(gist.get('publishDate'), 1000),
 254                 'formats': formats,
 255             }
 256             for k in ('categories', 'tags'):
 257                 info[k] = [v['title'] for v in content_data.get(k, []) if v.get('title')]
 258             return info
 259         else:
 260             film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
 261
 262             snag = self._parse_json(
 263                 self._search_regex(
 264                     r'Snag\.page\.data\s*=\s*(\[.+?\]);', webpage, 'snag', default='[]'),
 265                 display_id)
 266
 267             for item in snag:
 268                 if item.get('data', {}).get('film', {}).get('id') == film_id:
 269                     data = item['data']['film']
 270                     title = data['title']
 271                     description = clean_html(data.get('synopsis'))
 272                     thumbnail = data.get('image')
 273                     duration = int_or_none(data.get('duration') or data.get('runtime'))
 274                     categories = [
 275                         category['title'] for category in data.get('categories', [])
 276                         if category.get('title')]
 277                     break
 278             else:
 279                 title = self._html_search_regex(
 280                     (r'itemprop="title">([^<]+)<',
 281                      r'(?s)itemprop="title">(.+?)<div'), webpage, 'title')
 282                 description = self._html_search_regex(
 283                     r'(?s)<div itemprop="description" class="film-synopsis-inner ">(.+?)</div>',
 284                     webpage, 'description', default=None) or self._og_search_description(webpage)
 285                 thumbnail = self._og_search_thumbnail(webpage)
 286                 duration = parse_duration(self._search_regex(
 287                     r'<span itemprop="duration" class="film-duration strong">([^<]+)<',
 288                     webpage, 'duration', fatal=False))
 289                 categories = re.findall(r'<a href="/movies/[^"]+">([^<]+)</a>', webpage)
 290
 291             return {
 292                 '_type': 'url_transparent',
 293                 'url': 'http://%s/embed/player?filmId=%s' % (domain, film_id),
 294                 'id': film_id,
 295                 'display_id': display_id,
 296                 'title': title,
 297                 'description': description,
 298                 'thumbnail': thumbnail,
 299                 'duration': duration,
 300                 'categories': categories,
 301                 'ie_key': 'ViewLiftEmbed',
 302             }