]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/viewlift.py
d/p/skip-style-checks-for-flake8: Add metadata to please lintian.
[youtubedl] / youtube_dl / extractor / viewlift.py
1 from __future__ import unicode_literals
2
3 import base64
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_urllib_parse_unquote
8 from ..utils import (
9 ExtractorError,
10 clean_html,
11 determine_ext,
12 int_or_none,
13 js_to_json,
14 parse_age_limit,
15 parse_duration,
16 try_get,
17 )
18
19
20 class ViewLiftBaseIE(InfoExtractor):
21 _DOMAINS_REGEX = r'(?:(?:main\.)?snagfilms|snagxtreme|funnyforfree|kiddovid|winnersview|(?:monumental|lax)sportsnetwork|vayafilm)\.com|hoichoi\.tv'
22
23
24 class ViewLiftEmbedIE(ViewLiftBaseIE):
25 _VALID_URL = r'https?://(?:(?:www|embed)\.)?(?:%s)/embed/player\?.*\bfilmId=(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})' % ViewLiftBaseIE._DOMAINS_REGEX
26 _TESTS = [{
27 'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831&w=500',
28 'md5': '2924e9215c6eff7a55ed35b72276bd93',
29 'info_dict': {
30 'id': '74849a00-85a9-11e1-9660-123139220831',
31 'ext': 'mp4',
32 'title': '#whilewewatch',
33 }
34 }, {
35 # invalid labels, 360p is better that 480p
36 'url': 'http://www.snagfilms.com/embed/player?filmId=17ca0950-a74a-11e0-a92a-0026bb61d036',
37 'md5': '882fca19b9eb27ef865efeeaed376a48',
38 'info_dict': {
39 'id': '17ca0950-a74a-11e0-a92a-0026bb61d036',
40 'ext': 'mp4',
41 'title': 'Life in Limbo',
42 }
43 }, {
44 'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
45 'only_matching': True,
46 }]
47
48 @staticmethod
49 def _extract_url(webpage):
50 mobj = re.search(
51 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?(?:%s)/embed/player.+?)\1' % ViewLiftBaseIE._DOMAINS_REGEX,
52 webpage)
53 if mobj:
54 return mobj.group('url')
55
56 def _real_extract(self, url):
57 video_id = self._match_id(url)
58
59 webpage = self._download_webpage(url, video_id)
60
61 if '>This film is not playable in your area.<' in webpage:
62 raise ExtractorError(
63 'Film %s is not playable in your area.' % video_id, expected=True)
64
65 formats = []
66 has_bitrate = False
67 sources = self._parse_json(self._search_regex(
68 r'(?s)sources:\s*(\[.+?\]),', webpage,
69 'sources', default='[]'), video_id, js_to_json)
70 for source in sources:
71 file_ = source.get('file')
72 if not file_:
73 continue
74 type_ = source.get('type')
75 ext = determine_ext(file_)
76 format_id = source.get('label') or ext
77 if all(v in ('m3u8', 'hls') for v in (type_, ext)):
78 formats.extend(self._extract_m3u8_formats(
79 file_, video_id, 'mp4', 'm3u8_native',
80 m3u8_id='hls', fatal=False))
81 else:
82 bitrate = int_or_none(self._search_regex(
83 [r'(\d+)kbps', r'_\d{1,2}x\d{1,2}_(\d{3,})\.%s' % ext],
84 file_, 'bitrate', default=None))
85 if not has_bitrate and bitrate:
86 has_bitrate = True
87 height = int_or_none(self._search_regex(
88 r'^(\d+)[pP]$', format_id, 'height', default=None))
89 formats.append({
90 'url': file_,
91 'format_id': 'http-%s%s' % (format_id, ('-%dk' % bitrate if bitrate else '')),
92 'tbr': bitrate,
93 'height': height,
94 })
95 if not formats:
96 hls_url = self._parse_json(self._search_regex(
97 r'filmInfo\.src\s*=\s*({.+?});',
98 webpage, 'src'), video_id, js_to_json)['src']
99 formats = self._extract_m3u8_formats(
100 hls_url, video_id, 'mp4', 'm3u8_native',
101 m3u8_id='hls', fatal=False)
102 field_preference = None if has_bitrate else ('height', 'tbr', 'format_id')
103 self._sort_formats(formats, field_preference)
104
105 title = self._search_regex(
106 [r"title\s*:\s*'([^']+)'", r'<title>([^<]+)</title>'],
107 webpage, 'title')
108
109 return {
110 'id': video_id,
111 'title': title,
112 'formats': formats,
113 }
114
115
116 class ViewLiftIE(ViewLiftBaseIE):
117 _VALID_URL = r'https?://(?:www\.)?(?P<domain>%s)(?:/(?:films/title|show|(?:news/)?videos?))?/(?P<id>[^?#]+)' % ViewLiftBaseIE._DOMAINS_REGEX
118 _TESTS = [{
119 'url': 'http://www.snagfilms.com/films/title/lost_for_life',
120 'md5': '19844f897b35af219773fd63bdec2942',
121 'info_dict': {
122 'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
123 'display_id': 'lost_for_life',
124 'ext': 'mp4',
125 'title': 'Lost for Life',
126 'description': 'md5:ea10b5a50405ae1f7b5269a6ec594102',
127 'thumbnail': r're:^https?://.*\.jpg',
128 'duration': 4489,
129 'categories': 'mincount:3',
130 'age_limit': 14,
131 'upload_date': '20150421',
132 'timestamp': 1429656820,
133 }
134 }, {
135 'url': 'http://www.snagfilms.com/show/the_world_cut_project/india',
136 'md5': 'e6292e5b837642bbda82d7f8bf3fbdfd',
137 'info_dict': {
138 'id': '00000145-d75c-d96e-a9c7-ff5c67b20000',
139 'display_id': 'the_world_cut_project/india',
140 'ext': 'mp4',
141 'title': 'India',
142 'description': 'md5:5c168c5a8f4719c146aad2e0dfac6f5f',
143 'thumbnail': r're:^https?://.*\.jpg',
144 'duration': 979,
145 'timestamp': 1399478279,
146 'upload_date': '20140507',
147 }
148 }, {
149 'url': 'http://main.snagfilms.com/augie_alone/s_2_ep_12_love',
150 'info_dict': {
151 'id': '00000148-7b53-de26-a9fb-fbf306f70020',
152 'display_id': 'augie_alone/s_2_ep_12_love',
153 'ext': 'mp4',
154 'title': 'Augie, Alone:S. 2 Ep. 12 - Love',
155 'description': 'md5:db2a5c72d994f16a780c1eb353a8f403',
156 'thumbnail': r're:^https?://.*\.jpg',
157 'duration': 107,
158 },
159 'params': {
160 'skip_download': True,
161 },
162 }, {
163 'url': 'http://main.snagfilms.com/films/title/the_freebie',
164 'only_matching': True,
165 }, {
166 # Film is not playable in your area.
167 'url': 'http://www.snagfilms.com/films/title/inside_mecca',
168 'only_matching': True,
169 }, {
170 # Film is not available.
171 'url': 'http://www.snagfilms.com/show/augie_alone/flirting',
172 'only_matching': True,
173 }, {
174 'url': 'http://www.winnersview.com/videos/the-good-son',
175 'only_matching': True,
176 }, {
177 # Was once Kaltura embed
178 'url': 'https://www.monumentalsportsnetwork.com/videos/john-carlson-postgame-2-25-15',
179 'only_matching': True,
180 }]
181
182 @classmethod
183 def suitable(cls, url):
184 return False if ViewLiftEmbedIE.suitable(url) else super(ViewLiftIE, cls).suitable(url)
185
186 def _real_extract(self, url):
187 domain, display_id = re.match(self._VALID_URL, url).groups()
188
189 webpage = self._download_webpage(url, display_id)
190
191 if ">Sorry, the Film you're looking for is not available.<" in webpage:
192 raise ExtractorError(
193 'Film %s is not available.' % display_id, expected=True)
194
195 initial_store_state = self._search_regex(
196 r"window\.initialStoreState\s*=.*?JSON\.parse\(unescape\(atob\('([^']+)'\)\)\)",
197 webpage, 'Initial Store State', default=None)
198 if initial_store_state:
199 modules = self._parse_json(compat_urllib_parse_unquote(base64.b64decode(
200 initial_store_state).decode()), display_id)['page']['data']['modules']
201 content_data = next(m['contentData'][0] for m in modules if m.get('moduleType') == 'VideoDetailModule')
202 gist = content_data['gist']
203 film_id = gist['id']
204 title = gist['title']
205 video_assets = try_get(
206 content_data, lambda x: x['streamingInfo']['videoAssets'], dict)
207 if not video_assets:
208 token = self._download_json(
209 'https://prod-api.viewlift.com/identity/anonymous-token',
210 film_id, 'Downloading authorization token',
211 query={'site': 'snagfilms'})['authorizationToken']
212 video_assets = self._download_json(
213 'https://prod-api.viewlift.com/entitlement/video/status',
214 film_id, headers={
215 'Authorization': token,
216 'Referer': url,
217 }, query={
218 'id': film_id
219 })['video']['streamingInfo']['videoAssets']
220
221 formats = []
222 mpeg_video_assets = video_assets.get('mpeg') or []
223 for video_asset in mpeg_video_assets:
224 video_asset_url = video_asset.get('url')
225 if not video_asset:
226 continue
227 bitrate = int_or_none(video_asset.get('bitrate'))
228 height = int_or_none(self._search_regex(
229 r'^_?(\d+)[pP]$', video_asset.get('renditionValue'),
230 'height', default=None))
231 formats.append({
232 'url': video_asset_url,
233 'format_id': 'http%s' % ('-%d' % bitrate if bitrate else ''),
234 'tbr': bitrate,
235 'height': height,
236 'vcodec': video_asset.get('codec'),
237 })
238
239 hls_url = video_assets.get('hls')
240 if hls_url:
241 formats.extend(self._extract_m3u8_formats(
242 hls_url, film_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
243 self._sort_formats(formats, ('height', 'tbr', 'format_id'))
244
245 info = {
246 'id': film_id,
247 'display_id': display_id,
248 'title': title,
249 'description': gist.get('description'),
250 'thumbnail': gist.get('videoImageUrl'),
251 'duration': int_or_none(gist.get('runtime')),
252 'age_limit': parse_age_limit(content_data.get('parentalRating')),
253 'timestamp': int_or_none(gist.get('publishDate'), 1000),
254 'formats': formats,
255 }
256 for k in ('categories', 'tags'):
257 info[k] = [v['title'] for v in content_data.get(k, []) if v.get('title')]
258 return info
259 else:
260 film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
261
262 snag = self._parse_json(
263 self._search_regex(
264 r'Snag\.page\.data\s*=\s*(\[.+?\]);', webpage, 'snag', default='[]'),
265 display_id)
266
267 for item in snag:
268 if item.get('data', {}).get('film', {}).get('id') == film_id:
269 data = item['data']['film']
270 title = data['title']
271 description = clean_html(data.get('synopsis'))
272 thumbnail = data.get('image')
273 duration = int_or_none(data.get('duration') or data.get('runtime'))
274 categories = [
275 category['title'] for category in data.get('categories', [])
276 if category.get('title')]
277 break
278 else:
279 title = self._html_search_regex(
280 (r'itemprop="title">([^<]+)<',
281 r'(?s)itemprop="title">(.+?)<div'), webpage, 'title')
282 description = self._html_search_regex(
283 r'(?s)<div itemprop="description" class="film-synopsis-inner ">(.+?)</div>',
284 webpage, 'description', default=None) or self._og_search_description(webpage)
285 thumbnail = self._og_search_thumbnail(webpage)
286 duration = parse_duration(self._search_regex(
287 r'<span itemprop="duration" class="film-duration strong">([^<]+)<',
288 webpage, 'duration', fatal=False))
289 categories = re.findall(r'<a href="/movies/[^"]+">([^<]+)</a>', webpage)
290
291 return {
292 '_type': 'url_transparent',
293 'url': 'http://%s/embed/player?filmId=%s' % (domain, film_id),
294 'id': film_id,
295 'display_id': display_id,
296 'title': title,
297 'description': description,
298 'thumbnail': thumbnail,
299 'duration': duration,
300 'categories': categories,
301 'ie_key': 'ViewLiftEmbed',
302 }