]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/smotri.py
Prepare for upload.
[youtubedl] / youtube_dl / extractor / smotri.py
1 # encoding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5 import json
6 import hashlib
7 import uuid
8
9 from .common import InfoExtractor
10 from ..utils import (
11 compat_urllib_parse,
12 compat_urllib_request,
13 ExtractorError,
14 int_or_none,
15 unified_strdate,
16 )
17
18
19 class SmotriIE(InfoExtractor):
20 IE_DESC = 'Smotri.com'
21 IE_NAME = 'smotri'
22 _VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<id>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
23 _NETRC_MACHINE = 'smotri'
24
25 _TESTS = [
26 # real video id 2610366
27 {
28 'url': 'http://smotri.com/video/view/?id=v261036632ab',
29 'md5': '2a7b08249e6f5636557579c368040eb9',
30 'info_dict': {
31 'id': 'v261036632ab',
32 'ext': 'mp4',
33 'title': 'катастрофа с камер видеонаблюдения',
34 'uploader': 'rbc2008',
35 'uploader_id': 'rbc08',
36 'upload_date': '20131118',
37 'thumbnail': 'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg',
38 },
39 },
40 # real video id 57591
41 {
42 'url': 'http://smotri.com/video/view/?id=v57591cb20',
43 'md5': '830266dfc21f077eac5afd1883091bcd',
44 'info_dict': {
45 'id': 'v57591cb20',
46 'ext': 'flv',
47 'title': 'test',
48 'uploader': 'Support Photofile@photofile',
49 'uploader_id': 'support-photofile',
50 'upload_date': '20070704',
51 'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg',
52 },
53 },
54 # video-password
55 {
56 'url': 'http://smotri.com/video/view/?id=v1390466a13c',
57 'md5': 'f6331cef33cad65a0815ee482a54440b',
58 'info_dict': {
59 'id': 'v1390466a13c',
60 'ext': 'mp4',
61 'title': 'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1',
62 'uploader': 'timoxa40',
63 'uploader_id': 'timoxa40',
64 'upload_date': '20100404',
65 'thumbnail': 'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg',
66 },
67 'params': {
68 'videopassword': 'qwerty',
69 },
70 },
71 # age limit + video-password
72 {
73 'url': 'http://smotri.com/video/view/?id=v15408898bcf',
74 'md5': '91e909c9f0521adf5ee86fbe073aad70',
75 'info_dict': {
76 'id': 'v15408898bcf',
77 'ext': 'flv',
78 'title': 'этот ролик не покажут по ТВ',
79 'uploader': 'zzxxx',
80 'uploader_id': 'ueggb',
81 'upload_date': '20101001',
82 'thumbnail': 'http://frame3.loadup.ru/75/75/1540889.1.3.jpg',
83 'age_limit': 18,
84 },
85 'params': {
86 'videopassword': '333'
87 }
88 },
89 # swf player
90 {
91 'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',
92 'md5': '4d47034979d9390d14acdf59c4935bc2',
93 'info_dict': {
94 'id': 'v9188090500',
95 'ext': 'mp4',
96 'title': 'Shakira - Don\'t Bother',
97 'uploader': 'HannahL',
98 'uploader_id': 'lisaha95',
99 'upload_date': '20090331',
100 'thumbnail': 'http://frame8.loadup.ru/44/0b/918809.7.3.jpg',
101 },
102 },
103 ]
104
105 @classmethod
106 def _extract_url(cls, webpage):
107 mobj = re.search(
108 r'<embed[^>]src=(["\'])(?P<url>http://pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=v.+?\1)',
109 webpage)
110 if mobj is not None:
111 return mobj.group('url')
112
113 mobj = re.search(
114 r'''(?x)<div\s+class="video_file">http://smotri\.com/video/download/file/[^<]+</div>\s*
115 <div\s+class="video_image">[^<]+</div>\s*
116 <div\s+class="video_id">(?P<id>[^<]+)</div>''', webpage)
117 if mobj is not None:
118 return 'http://smotri.com/video/view/?id=%s' % mobj.group('id')
119
120 def _search_meta(self, name, html, display_name=None):
121 if display_name is None:
122 display_name = name
123 return self._html_search_regex(
124 r'<meta itemprop="%s" content="([^"]+)" />' % re.escape(name),
125 html, display_name, fatal=False)
126 return self._html_search_meta(name, html, display_name)
127
128 def _real_extract(self, url):
129 video_id = self._match_id(url)
130
131 video_form = {
132 'ticket': video_id,
133 'video_url': '1',
134 'frame_url': '1',
135 'devid': 'LoadupFlashPlayer',
136 'getvideoinfo': '1',
137 }
138
139 request = compat_urllib_request.Request(
140 'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
141 request.add_header('Content-Type', 'application/x-www-form-urlencoded')
142
143 video = self._download_json(request, video_id, 'Downloading video JSON')
144
145 if video.get('_moderate_no') or not video.get('moderated'):
146 raise ExtractorError('Video %s has not been approved by moderator' % video_id, expected=True)
147
148 if video.get('error'):
149 raise ExtractorError('Video %s does not exist' % video_id, expected=True)
150
151 video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
152 title = video['title']
153 thumbnail = video['_imgURL']
154 upload_date = unified_strdate(video['added'])
155 uploader = video['userNick']
156 uploader_id = video['userLogin']
157 duration = int_or_none(video['duration'])
158
159 # Video JSON does not provide enough meta data
160 # We will extract some from the video web page instead
161 webpage_url = 'http://smotri.com/video/view/?id=%s' % video_id
162 webpage = self._download_webpage(webpage_url, video_id, 'Downloading video page')
163
164 # Warning if video is unavailable
165 warning = self._html_search_regex(
166 r'<div class="videoUnModer">(.*?)</div>', webpage,
167 'warning message', default=None)
168 if warning is not None:
169 self._downloader.report_warning(
170 'Video %s may not be available; smotri said: %s ' %
171 (video_id, warning))
172
173 # Adult content
174 if re.search('EroConfirmText">', webpage) is not None:
175 self.report_age_confirmation()
176 confirm_string = self._html_search_regex(
177 r'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id,
178 webpage, 'confirm string')
179 confirm_url = webpage_url + '&confirm=%s' % confirm_string
180 webpage = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)')
181 adult_content = True
182 else:
183 adult_content = False
184
185 view_count = self._html_search_regex(
186 'Общее количество просмотров.*?<span class="Number">(\\d+)</span>',
187 webpage, 'view count', fatal=False, flags=re.MULTILINE | re.DOTALL)
188
189 return {
190 'id': video_id,
191 'url': video_url,
192 'title': title,
193 'thumbnail': thumbnail,
194 'uploader': uploader,
195 'upload_date': upload_date,
196 'uploader_id': uploader_id,
197 'duration': duration,
198 'view_count': int_or_none(view_count),
199 'age_limit': 18 if adult_content else 0,
200 }
201
202
203 class SmotriCommunityIE(InfoExtractor):
204 IE_DESC = 'Smotri.com community videos'
205 IE_NAME = 'smotri:community'
206 _VALID_URL = r'^https?://(?:www\.)?smotri\.com/community/video/(?P<communityid>[0-9A-Za-z_\'-]+)'
207 _TEST = {
208 'url': 'http://smotri.com/community/video/kommuna',
209 'info_dict': {
210 'id': 'kommuna',
211 'title': 'КПРФ',
212 },
213 'playlist_mincount': 4,
214 }
215
216 def _real_extract(self, url):
217 mobj = re.match(self._VALID_URL, url)
218 community_id = mobj.group('communityid')
219
220 url = 'http://smotri.com/export/rss/video/by/community/-/%s/video.xml' % community_id
221 rss = self._download_xml(url, community_id, 'Downloading community RSS')
222
223 entries = [self.url_result(video_url.text, 'Smotri')
224 for video_url in rss.findall('./channel/item/link')]
225
226 description_text = rss.find('./channel/description').text
227 community_title = self._html_search_regex(
228 '^Видео сообщества "([^"]+)"$', description_text, 'community title')
229
230 return self.playlist_result(entries, community_id, community_title)
231
232
233 class SmotriUserIE(InfoExtractor):
234 IE_DESC = 'Smotri.com user videos'
235 IE_NAME = 'smotri:user'
236 _VALID_URL = r'^https?://(?:www\.)?smotri\.com/user/(?P<userid>[0-9A-Za-z_\'-]+)'
237 _TESTS = [{
238 'url': 'http://smotri.com/user/inspector',
239 'info_dict': {
240 'id': 'inspector',
241 'title': 'Inspector',
242 },
243 'playlist_mincount': 9,
244 }]
245
246 def _real_extract(self, url):
247 mobj = re.match(self._VALID_URL, url)
248 user_id = mobj.group('userid')
249
250 url = 'http://smotri.com/export/rss/user/video/-/%s/video.xml' % user_id
251 rss = self._download_xml(url, user_id, 'Downloading user RSS')
252
253 entries = [self.url_result(video_url.text, 'Smotri')
254 for video_url in rss.findall('./channel/item/link')]
255
256 description_text = rss.find('./channel/description').text
257 user_nickname = self._html_search_regex(
258 '^Видео режиссера (.*)$', description_text,
259 'user nickname')
260
261 return self.playlist_result(entries, user_id, user_nickname)
262
263
264 class SmotriBroadcastIE(InfoExtractor):
265 IE_DESC = 'Smotri.com broadcasts'
266 IE_NAME = 'smotri:broadcast'
267 _VALID_URL = r'^https?://(?:www\.)?(?P<url>smotri\.com/live/(?P<broadcastid>[^/]+))/?.*'
268
269 def _real_extract(self, url):
270 mobj = re.match(self._VALID_URL, url)
271 broadcast_id = mobj.group('broadcastid')
272
273 broadcast_url = 'http://' + mobj.group('url')
274 broadcast_page = self._download_webpage(broadcast_url, broadcast_id, 'Downloading broadcast page')
275
276 if re.search('>Режиссер с логином <br/>"%s"<br/> <span>не существует<' % broadcast_id, broadcast_page) is not None:
277 raise ExtractorError('Broadcast %s does not exist' % broadcast_id, expected=True)
278
279 # Adult content
280 if re.search('EroConfirmText">', broadcast_page) is not None:
281
282 (username, password) = self._get_login_info()
283 if username is None:
284 raise ExtractorError('Erotic broadcasts allowed only for registered users, '
285 'use --username and --password options to provide account credentials.', expected=True)
286
287 login_form = {
288 'login-hint53': '1',
289 'confirm_erotic': '1',
290 'login': username,
291 'password': password,
292 }
293
294 request = compat_urllib_request.Request(broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
295 request.add_header('Content-Type', 'application/x-www-form-urlencoded')
296 broadcast_page = self._download_webpage(request, broadcast_id, 'Logging in and confirming age')
297
298 if re.search('>Неверный логин или пароль<', broadcast_page) is not None:
299 raise ExtractorError('Unable to log in: bad username or password', expected=True)
300
301 adult_content = True
302 else:
303 adult_content = False
304
305 ticket = self._html_search_regex(
306 'window\.broadcast_control\.addFlashVar\\(\'file\', \'([^\']+)\'\\);',
307 broadcast_page, 'broadcast ticket')
308
309 url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket
310
311 broadcast_password = self._downloader.params.get('videopassword', None)
312 if broadcast_password:
313 url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest()
314
315 broadcast_json_page = self._download_webpage(url, broadcast_id, 'Downloading broadcast JSON')
316
317 try:
318 broadcast_json = json.loads(broadcast_json_page)
319
320 protected_broadcast = broadcast_json['_pass_protected'] == 1
321 if protected_broadcast and not broadcast_password:
322 raise ExtractorError('This broadcast is protected by a password, use the --video-password option', expected=True)
323
324 broadcast_offline = broadcast_json['is_play'] == 0
325 if broadcast_offline:
326 raise ExtractorError('Broadcast %s is offline' % broadcast_id, expected=True)
327
328 rtmp_url = broadcast_json['_server']
329 if not rtmp_url.startswith('rtmp://'):
330 raise ExtractorError('Unexpected broadcast rtmp URL')
331
332 broadcast_playpath = broadcast_json['_streamName']
333 broadcast_thumbnail = broadcast_json['_imgURL']
334 broadcast_title = broadcast_json['title']
335 broadcast_description = broadcast_json['description']
336 broadcaster_nick = broadcast_json['nick']
337 broadcaster_login = broadcast_json['login']
338 rtmp_conn = 'S:%s' % uuid.uuid4().hex
339 except KeyError:
340 if protected_broadcast:
341 raise ExtractorError('Bad broadcast password', expected=True)
342 raise ExtractorError('Unexpected broadcast JSON')
343
344 return {
345 'id': broadcast_id,
346 'url': rtmp_url,
347 'title': broadcast_title,
348 'thumbnail': broadcast_thumbnail,
349 'description': broadcast_description,
350 'uploader': broadcaster_nick,
351 'uploader_id': broadcaster_login,
352 'age_limit': 18 if adult_content else 0,
353 'ext': 'flv',
354 'play_path': broadcast_playpath,
355 'rtmp_live': True,
356 'rtmp_conn': rtmp_conn
357 }