Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/viewlift.py

   1 from __future__ import unicode_literals
   2
   3 import base64
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import compat_urllib_parse_unquote
   8 from ..utils import (
   9     ExtractorError,
  10     clean_html,
  11     determine_ext,
  12     int_or_none,
  13     js_to_json,
  14     parse_age_limit,
  15     parse_duration,
  16 )
  17
  18
  19 class ViewLiftBaseIE(InfoExtractor):
  20     _DOMAINS_REGEX = r'(?:snagfilms|snagxtreme|funnyforfree|kiddovid|winnersview|(?:monumental|lax)sportsnetwork|vayafilm)\.com|hoichoi\.tv'
  21
  22
  23 class ViewLiftEmbedIE(ViewLiftBaseIE):
  24     _VALID_URL = r'https?://(?:(?:www|embed)\.)?(?:%s)/embed/player\?.*\bfilmId=(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})' % ViewLiftBaseIE._DOMAINS_REGEX
  25     _TESTS = [{
  26         'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831&w=500',
  27         'md5': '2924e9215c6eff7a55ed35b72276bd93',
  28         'info_dict': {
  29             'id': '74849a00-85a9-11e1-9660-123139220831',
  30             'ext': 'mp4',
  31             'title': '#whilewewatch',
  32         }
  33     }, {
  34         # invalid labels, 360p is better that 480p
  35         'url': 'http://www.snagfilms.com/embed/player?filmId=17ca0950-a74a-11e0-a92a-0026bb61d036',
  36         'md5': '882fca19b9eb27ef865efeeaed376a48',
  37         'info_dict': {
  38             'id': '17ca0950-a74a-11e0-a92a-0026bb61d036',
  39             'ext': 'mp4',
  40             'title': 'Life in Limbo',
  41         }
  42     }, {
  43         'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
  44         'only_matching': True,
  45     }]
  46
  47     @staticmethod
  48     def _extract_url(webpage):
  49         mobj = re.search(
  50             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?(?:%s)/embed/player.+?)\1' % ViewLiftBaseIE._DOMAINS_REGEX,
  51             webpage)
  52         if mobj:
  53             return mobj.group('url')
  54
  55     def _real_extract(self, url):
  56         video_id = self._match_id(url)
  57
  58         webpage = self._download_webpage(url, video_id)
  59
  60         if '>This film is not playable in your area.<' in webpage:
  61             raise ExtractorError(
  62                 'Film %s is not playable in your area.' % video_id, expected=True)
  63
  64         formats = []
  65         has_bitrate = False
  66         sources = self._parse_json(self._search_regex(
  67             r'(?s)sources:\s*(\[.+?\]),', webpage,
  68             'sources', default='[]'), video_id, js_to_json)
  69         for source in sources:
  70             file_ = source.get('file')
  71             if not file_:
  72                 continue
  73             type_ = source.get('type')
  74             ext = determine_ext(file_)
  75             format_id = source.get('label') or ext
  76             if all(v in ('m3u8', 'hls') for v in (type_, ext)):
  77                 formats.extend(self._extract_m3u8_formats(
  78                     file_, video_id, 'mp4', 'm3u8_native',
  79                     m3u8_id='hls', fatal=False))
  80             else:
  81                 bitrate = int_or_none(self._search_regex(
  82                     [r'(\d+)kbps', r'_\d{1,2}x\d{1,2}_(\d{3,})\.%s' % ext],
  83                     file_, 'bitrate', default=None))
  84                 if not has_bitrate and bitrate:
  85                     has_bitrate = True
  86                 height = int_or_none(self._search_regex(
  87                     r'^(\d+)[pP]$', format_id, 'height', default=None))
  88                 formats.append({
  89                     'url': file_,
  90                     'format_id': 'http-%s%s' % (format_id, ('-%dk' % bitrate if bitrate else '')),
  91                     'tbr': bitrate,
  92                     'height': height,
  93                 })
  94         if not formats:
  95             hls_url = self._parse_json(self._search_regex(
  96                 r'filmInfo\.src\s*=\s*({.+?});',
  97                 webpage, 'src'), video_id, js_to_json)['src']
  98             formats = self._extract_m3u8_formats(
  99                 hls_url, video_id, 'mp4', 'm3u8_native',
 100                 m3u8_id='hls', fatal=False)
 101         field_preference = None if has_bitrate else ('height', 'tbr', 'format_id')
 102         self._sort_formats(formats, field_preference)
 103
 104         title = self._search_regex(
 105             [r"title\s*:\s*'([^']+)'", r'<title>([^<]+)</title>'],
 106             webpage, 'title')
 107
 108         return {
 109             'id': video_id,
 110             'title': title,
 111             'formats': formats,
 112         }
 113
 114
 115 class ViewLiftIE(ViewLiftBaseIE):
 116     _VALID_URL = r'https?://(?:www\.)?(?P<domain>%s)/(?:films/title|show|(?:news/)?videos?)/(?P<id>[^?#]+)' % ViewLiftBaseIE._DOMAINS_REGEX
 117     _TESTS = [{
 118         'url': 'http://www.snagfilms.com/films/title/lost_for_life',
 119         'md5': '19844f897b35af219773fd63bdec2942',
 120         'info_dict': {
 121             'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
 122             'display_id': 'lost_for_life',
 123             'ext': 'mp4',
 124             'title': 'Lost for Life',
 125             'description': 'md5:ea10b5a50405ae1f7b5269a6ec594102',
 126             'thumbnail': r're:^https?://.*\.jpg',
 127             'duration': 4489,
 128             'categories': 'mincount:3',
 129             'age_limit': 14,
 130             'upload_date': '20150421',
 131             'timestamp': 1429656819,
 132         }
 133     }, {
 134         'url': 'http://www.snagfilms.com/show/the_world_cut_project/india',
 135         'md5': 'e6292e5b837642bbda82d7f8bf3fbdfd',
 136         'info_dict': {
 137             'id': '00000145-d75c-d96e-a9c7-ff5c67b20000',
 138             'display_id': 'the_world_cut_project/india',
 139             'ext': 'mp4',
 140             'title': 'India',
 141             'description': 'md5:5c168c5a8f4719c146aad2e0dfac6f5f',
 142             'thumbnail': r're:^https?://.*\.jpg',
 143             'duration': 979,
 144             'categories': 'mincount:2',
 145             'timestamp': 1399478279,
 146             'upload_date': '20140507',
 147         }
 148     }, {
 149         # Film is not playable in your area.
 150         'url': 'http://www.snagfilms.com/films/title/inside_mecca',
 151         'only_matching': True,
 152     }, {
 153         # Film is not available.
 154         'url': 'http://www.snagfilms.com/show/augie_alone/flirting',
 155         'only_matching': True,
 156     }, {
 157         'url': 'http://www.winnersview.com/videos/the-good-son',
 158         'only_matching': True,
 159     }, {
 160         # Was once Kaltura embed
 161         'url': 'https://www.monumentalsportsnetwork.com/videos/john-carlson-postgame-2-25-15',
 162         'only_matching': True,
 163     }]
 164
 165     def _real_extract(self, url):
 166         domain, display_id = re.match(self._VALID_URL, url).groups()
 167
 168         webpage = self._download_webpage(url, display_id)
 169
 170         if ">Sorry, the Film you're looking for is not available.<" in webpage:
 171             raise ExtractorError(
 172                 'Film %s is not available.' % display_id, expected=True)
 173
 174         initial_store_state = self._search_regex(
 175             r"window\.initialStoreState\s*=.*?JSON\.parse\(unescape\(atob\('([^']+)'\)\)\)",
 176             webpage, 'Initial Store State', default=None)
 177         if initial_store_state:
 178             modules = self._parse_json(compat_urllib_parse_unquote(base64.b64decode(
 179                 initial_store_state).decode()), display_id)['page']['data']['modules']
 180             content_data = next(m['contentData'][0] for m in modules if m.get('moduleType') == 'VideoDetailModule')
 181             gist = content_data['gist']
 182             film_id = gist['id']
 183             title = gist['title']
 184             video_assets = content_data['streamingInfo']['videoAssets']
 185
 186             formats = []
 187             mpeg_video_assets = video_assets.get('mpeg') or []
 188             for video_asset in mpeg_video_assets:
 189                 video_asset_url = video_asset.get('url')
 190                 if not video_asset:
 191                     continue
 192                 bitrate = int_or_none(video_asset.get('bitrate'))
 193                 height = int_or_none(self._search_regex(
 194                     r'^_?(\d+)[pP]$', video_asset.get('renditionValue'),
 195                     'height', default=None))
 196                 formats.append({
 197                     'url': video_asset_url,
 198                     'format_id': 'http%s' % ('-%d' % bitrate if bitrate else ''),
 199                     'tbr': bitrate,
 200                     'height': height,
 201                     'vcodec': video_asset.get('codec'),
 202                 })
 203
 204             hls_url = video_assets.get('hls')
 205             if hls_url:
 206                 formats.extend(self._extract_m3u8_formats(
 207                     hls_url, film_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
 208             self._sort_formats(formats, ('height', 'tbr', 'format_id'))
 209
 210             info = {
 211                 'id': film_id,
 212                 'display_id': display_id,
 213                 'title': title,
 214                 'description': gist.get('description'),
 215                 'thumbnail': gist.get('videoImageUrl'),
 216                 'duration': int_or_none(gist.get('runtime')),
 217                 'age_limit': parse_age_limit(content_data.get('parentalRating')),
 218                 'timestamp': int_or_none(gist.get('publishDate'), 1000),
 219                 'formats': formats,
 220             }
 221             for k in ('categories', 'tags'):
 222                 info[k] = [v['title'] for v in content_data.get(k, []) if v.get('title')]
 223             return info
 224         else:
 225             film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
 226
 227             snag = self._parse_json(
 228                 self._search_regex(
 229                     r'Snag\.page\.data\s*=\s*(\[.+?\]);', webpage, 'snag', default='[]'),
 230                 display_id)
 231
 232             for item in snag:
 233                 if item.get('data', {}).get('film', {}).get('id') == film_id:
 234                     data = item['data']['film']
 235                     title = data['title']
 236                     description = clean_html(data.get('synopsis'))
 237                     thumbnail = data.get('image')
 238                     duration = int_or_none(data.get('duration') or data.get('runtime'))
 239                     categories = [
 240                         category['title'] for category in data.get('categories', [])
 241                         if category.get('title')]
 242                     break
 243             else:
 244                 title = self._search_regex(
 245                     r'itemprop="title">([^<]+)<', webpage, 'title')
 246                 description = self._html_search_regex(
 247                     r'(?s)<div itemprop="description" class="film-synopsis-inner ">(.+?)</div>',
 248                     webpage, 'description', default=None) or self._og_search_description(webpage)
 249                 thumbnail = self._og_search_thumbnail(webpage)
 250                 duration = parse_duration(self._search_regex(
 251                     r'<span itemprop="duration" class="film-duration strong">([^<]+)<',
 252                     webpage, 'duration', fatal=False))
 253                 categories = re.findall(r'<a href="/movies/[^"]+">([^<]+)</a>', webpage)
 254
 255             return {
 256                 '_type': 'url_transparent',
 257                 'url': 'http://%s/embed/player?filmId=%s' % (domain, film_id),
 258                 'id': film_id,
 259                 'display_id': display_id,
 260                 'title': title,
 261                 'description': description,
 262                 'thumbnail': thumbnail,
 263                 'duration': duration,
 264                 'categories': categories,
 265                 'ie_key': 'ViewLiftEmbed',
 266             }