]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/viewlift.py
Prepare new version
[youtubedl] / youtube_dl / extractor / viewlift.py
1 from __future__ import unicode_literals
2
3 import base64
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_urllib_parse_unquote
8 from ..utils import (
9 ExtractorError,
10 clean_html,
11 determine_ext,
12 int_or_none,
13 js_to_json,
14 parse_age_limit,
15 parse_duration,
16 )
17
18
19 class ViewLiftBaseIE(InfoExtractor):
20 _DOMAINS_REGEX = r'(?:snagfilms|snagxtreme|funnyforfree|kiddovid|winnersview|(?:monumental|lax)sportsnetwork|vayafilm)\.com|hoichoi\.tv'
21
22
23 class ViewLiftEmbedIE(ViewLiftBaseIE):
24 _VALID_URL = r'https?://(?:(?:www|embed)\.)?(?:%s)/embed/player\?.*\bfilmId=(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})' % ViewLiftBaseIE._DOMAINS_REGEX
25 _TESTS = [{
26 'url': 'http://embed.snagfilms.com/embed/player?filmId=74849a00-85a9-11e1-9660-123139220831&w=500',
27 'md5': '2924e9215c6eff7a55ed35b72276bd93',
28 'info_dict': {
29 'id': '74849a00-85a9-11e1-9660-123139220831',
30 'ext': 'mp4',
31 'title': '#whilewewatch',
32 }
33 }, {
34 # invalid labels, 360p is better that 480p
35 'url': 'http://www.snagfilms.com/embed/player?filmId=17ca0950-a74a-11e0-a92a-0026bb61d036',
36 'md5': '882fca19b9eb27ef865efeeaed376a48',
37 'info_dict': {
38 'id': '17ca0950-a74a-11e0-a92a-0026bb61d036',
39 'ext': 'mp4',
40 'title': 'Life in Limbo',
41 }
42 }, {
43 'url': 'http://www.snagfilms.com/embed/player?filmId=0000014c-de2f-d5d6-abcf-ffef58af0017',
44 'only_matching': True,
45 }]
46
47 @staticmethod
48 def _extract_url(webpage):
49 mobj = re.search(
50 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:embed\.)?(?:%s)/embed/player.+?)\1' % ViewLiftBaseIE._DOMAINS_REGEX,
51 webpage)
52 if mobj:
53 return mobj.group('url')
54
55 def _real_extract(self, url):
56 video_id = self._match_id(url)
57
58 webpage = self._download_webpage(url, video_id)
59
60 if '>This film is not playable in your area.<' in webpage:
61 raise ExtractorError(
62 'Film %s is not playable in your area.' % video_id, expected=True)
63
64 formats = []
65 has_bitrate = False
66 sources = self._parse_json(self._search_regex(
67 r'(?s)sources:\s*(\[.+?\]),', webpage,
68 'sources', default='[]'), video_id, js_to_json)
69 for source in sources:
70 file_ = source.get('file')
71 if not file_:
72 continue
73 type_ = source.get('type')
74 ext = determine_ext(file_)
75 format_id = source.get('label') or ext
76 if all(v in ('m3u8', 'hls') for v in (type_, ext)):
77 formats.extend(self._extract_m3u8_formats(
78 file_, video_id, 'mp4', 'm3u8_native',
79 m3u8_id='hls', fatal=False))
80 else:
81 bitrate = int_or_none(self._search_regex(
82 [r'(\d+)kbps', r'_\d{1,2}x\d{1,2}_(\d{3,})\.%s' % ext],
83 file_, 'bitrate', default=None))
84 if not has_bitrate and bitrate:
85 has_bitrate = True
86 height = int_or_none(self._search_regex(
87 r'^(\d+)[pP]$', format_id, 'height', default=None))
88 formats.append({
89 'url': file_,
90 'format_id': 'http-%s%s' % (format_id, ('-%dk' % bitrate if bitrate else '')),
91 'tbr': bitrate,
92 'height': height,
93 })
94 if not formats:
95 hls_url = self._parse_json(self._search_regex(
96 r'filmInfo\.src\s*=\s*({.+?});',
97 webpage, 'src'), video_id, js_to_json)['src']
98 formats = self._extract_m3u8_formats(
99 hls_url, video_id, 'mp4', 'm3u8_native',
100 m3u8_id='hls', fatal=False)
101 field_preference = None if has_bitrate else ('height', 'tbr', 'format_id')
102 self._sort_formats(formats, field_preference)
103
104 title = self._search_regex(
105 [r"title\s*:\s*'([^']+)'", r'<title>([^<]+)</title>'],
106 webpage, 'title')
107
108 return {
109 'id': video_id,
110 'title': title,
111 'formats': formats,
112 }
113
114
115 class ViewLiftIE(ViewLiftBaseIE):
116 _VALID_URL = r'https?://(?:www\.)?(?P<domain>%s)/(?:films/title|show|(?:news/)?videos?)/(?P<id>[^?#]+)' % ViewLiftBaseIE._DOMAINS_REGEX
117 _TESTS = [{
118 'url': 'http://www.snagfilms.com/films/title/lost_for_life',
119 'md5': '19844f897b35af219773fd63bdec2942',
120 'info_dict': {
121 'id': '0000014c-de2f-d5d6-abcf-ffef58af0017',
122 'display_id': 'lost_for_life',
123 'ext': 'mp4',
124 'title': 'Lost for Life',
125 'description': 'md5:ea10b5a50405ae1f7b5269a6ec594102',
126 'thumbnail': r're:^https?://.*\.jpg',
127 'duration': 4489,
128 'categories': 'mincount:3',
129 'age_limit': 14,
130 'upload_date': '20150421',
131 'timestamp': 1429656819,
132 }
133 }, {
134 'url': 'http://www.snagfilms.com/show/the_world_cut_project/india',
135 'md5': 'e6292e5b837642bbda82d7f8bf3fbdfd',
136 'info_dict': {
137 'id': '00000145-d75c-d96e-a9c7-ff5c67b20000',
138 'display_id': 'the_world_cut_project/india',
139 'ext': 'mp4',
140 'title': 'India',
141 'description': 'md5:5c168c5a8f4719c146aad2e0dfac6f5f',
142 'thumbnail': r're:^https?://.*\.jpg',
143 'duration': 979,
144 'categories': 'mincount:2',
145 'timestamp': 1399478279,
146 'upload_date': '20140507',
147 }
148 }, {
149 # Film is not playable in your area.
150 'url': 'http://www.snagfilms.com/films/title/inside_mecca',
151 'only_matching': True,
152 }, {
153 # Film is not available.
154 'url': 'http://www.snagfilms.com/show/augie_alone/flirting',
155 'only_matching': True,
156 }, {
157 'url': 'http://www.winnersview.com/videos/the-good-son',
158 'only_matching': True,
159 }, {
160 # Was once Kaltura embed
161 'url': 'https://www.monumentalsportsnetwork.com/videos/john-carlson-postgame-2-25-15',
162 'only_matching': True,
163 }]
164
165 def _real_extract(self, url):
166 domain, display_id = re.match(self._VALID_URL, url).groups()
167
168 webpage = self._download_webpage(url, display_id)
169
170 if ">Sorry, the Film you're looking for is not available.<" in webpage:
171 raise ExtractorError(
172 'Film %s is not available.' % display_id, expected=True)
173
174 initial_store_state = self._search_regex(
175 r"window\.initialStoreState\s*=.*?JSON\.parse\(unescape\(atob\('([^']+)'\)\)\)",
176 webpage, 'Initial Store State', default=None)
177 if initial_store_state:
178 modules = self._parse_json(compat_urllib_parse_unquote(base64.b64decode(
179 initial_store_state).decode()), display_id)['page']['data']['modules']
180 content_data = next(m['contentData'][0] for m in modules if m.get('moduleType') == 'VideoDetailModule')
181 gist = content_data['gist']
182 film_id = gist['id']
183 title = gist['title']
184 video_assets = content_data['streamingInfo']['videoAssets']
185
186 formats = []
187 mpeg_video_assets = video_assets.get('mpeg') or []
188 for video_asset in mpeg_video_assets:
189 video_asset_url = video_asset.get('url')
190 if not video_asset:
191 continue
192 bitrate = int_or_none(video_asset.get('bitrate'))
193 height = int_or_none(self._search_regex(
194 r'^_?(\d+)[pP]$', video_asset.get('renditionValue'),
195 'height', default=None))
196 formats.append({
197 'url': video_asset_url,
198 'format_id': 'http%s' % ('-%d' % bitrate if bitrate else ''),
199 'tbr': bitrate,
200 'height': height,
201 'vcodec': video_asset.get('codec'),
202 })
203
204 hls_url = video_assets.get('hls')
205 if hls_url:
206 formats.extend(self._extract_m3u8_formats(
207 hls_url, film_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
208 self._sort_formats(formats, ('height', 'tbr', 'format_id'))
209
210 info = {
211 'id': film_id,
212 'display_id': display_id,
213 'title': title,
214 'description': gist.get('description'),
215 'thumbnail': gist.get('videoImageUrl'),
216 'duration': int_or_none(gist.get('runtime')),
217 'age_limit': parse_age_limit(content_data.get('parentalRating')),
218 'timestamp': int_or_none(gist.get('publishDate'), 1000),
219 'formats': formats,
220 }
221 for k in ('categories', 'tags'):
222 info[k] = [v['title'] for v in content_data.get(k, []) if v.get('title')]
223 return info
224 else:
225 film_id = self._search_regex(r'filmId=([\da-f-]{36})"', webpage, 'film id')
226
227 snag = self._parse_json(
228 self._search_regex(
229 r'Snag\.page\.data\s*=\s*(\[.+?\]);', webpage, 'snag', default='[]'),
230 display_id)
231
232 for item in snag:
233 if item.get('data', {}).get('film', {}).get('id') == film_id:
234 data = item['data']['film']
235 title = data['title']
236 description = clean_html(data.get('synopsis'))
237 thumbnail = data.get('image')
238 duration = int_or_none(data.get('duration') or data.get('runtime'))
239 categories = [
240 category['title'] for category in data.get('categories', [])
241 if category.get('title')]
242 break
243 else:
244 title = self._search_regex(
245 r'itemprop="title">([^<]+)<', webpage, 'title')
246 description = self._html_search_regex(
247 r'(?s)<div itemprop="description" class="film-synopsis-inner ">(.+?)</div>',
248 webpage, 'description', default=None) or self._og_search_description(webpage)
249 thumbnail = self._og_search_thumbnail(webpage)
250 duration = parse_duration(self._search_regex(
251 r'<span itemprop="duration" class="film-duration strong">([^<]+)<',
252 webpage, 'duration', fatal=False))
253 categories = re.findall(r'<a href="/movies/[^"]+">([^<]+)</a>', webpage)
254
255 return {
256 '_type': 'url_transparent',
257 'url': 'http://%s/embed/player?filmId=%s' % (domain, film_id),
258 'id': film_id,
259 'display_id': display_id,
260 'title': title,
261 'description': description,
262 'thumbnail': thumbnail,
263 'duration': duration,
264 'categories': categories,
265 'ie_key': 'ViewLiftEmbed',
266 }