]>
Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/vk.py
b8ea50362fc83f9bcdb480a9625d0dc55cd28b52
   2 from __future__ 
import unicode_literals
 
   8 from .common 
import InfoExtractor
 
  25 from .dailymotion 
import DailymotionIE
 
  26 from .pladform 
import PladformIE
 
  27 from .vimeo 
import VimeoIE
 
  28 from .youtube 
import YoutubeIE
 
  31 class VKBaseIE(InfoExtractor
): 
  35         (username
, password
) = self
._get
_login
_info
() 
  39         login_page
, url_handle 
= self
._download
_webpage
_handle
( 
  40             'https://vk.com', None, 'Downloading login page') 
  42         login_form 
= self
._hidden
_inputs
(login_page
) 
  45             'email': username
.encode('cp1251'), 
  46             'pass': password
.encode('cp1251'), 
  49         # https://new.vk.com/ serves two same remixlhk cookies in Set-Cookie header 
  50         # and expects the first one to be set rather than second (see 
  51         # https://github.com/rg3/youtube-dl/issues/9841#issuecomment-227871201). 
  52         # As of RFC6265 the newer one cookie should be set into cookie store 
  53         # what actually happens. 
  54         # We will workaround this VK issue by resetting the remixlhk cookie to 
  55         # the first one manually. 
  56         for header
, cookies 
in url_handle
.headers
.items(): 
  57             if header
.lower() != 'set-cookie': 
  59             if sys
.version_info
[0] >= 3: 
  60                 cookies 
= cookies
.encode('iso-8859-1') 
  61             cookies 
= cookies
.decode('utf-8') 
  62             remixlhk 
= re
.search(r
'remixlhk=(.+?);.*?\bdomain=(.+?)(?:[,;]|$)', cookies
) 
  64                 value
, domain 
= remixlhk
.groups() 
  65                 self
._set
_cookie
(domain
, 'remixlhk', value
) 
  68         login_page 
= self
._download
_webpage
( 
  69             'https://login.vk.com/?act=login', None, 
  71             data
=urlencode_postdata(login_form
)) 
  73         if re
.search(r
'onLoginFailed', login_page
): 
  75                 'Unable to login, incorrect username and/or password', expected
=True) 
  77     def _real_initialize(self
): 
  88                                 (?:(?:m|new)\.)?vk\.com/video_| 
  91                             ext\.php\?(?P<embed_query>.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+).*)| 
  93                                 (?:(?:m|new)\.)?vk\.com/(?:.+?\?.*?z=)?video| 
  94                                 (?:www\.)?daxab.com/embed/ 
  96                             (?P<videoid>-?\d+_\d+)(?:.*\blist=(?P<list_id>[\da-f]+))? 
 101             'url': 'http://vk.com/videos-77521?z=video-77521_162222515%2Fclub77521', 
 102             'md5': '0deae91935c54e00003c2a00646315f0', 
 106                 'title': 'ProtivoGunz - Хуёвая песня', 
 107                 'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*', 
 109                 'timestamp': 1329060660, 
 110                 'upload_date': '20120212', 
 115             'url': 'http://vk.com/video205387401_165548505', 
 116             'md5': '6c0aeb2e90396ba97035b9cbde548700', 
 120                 'uploader': 'Tom Cruise', 
 123                 'timestamp': 1374374880, 
 124                 'upload_date': '20130721', 
 129             'note': 'Embedded video', 
 130             'url': 'http://vk.com/video_ext.php?oid=32194266&id=162925554&hash=7d8c2e0d5e05aeaa&hd=1', 
 131             'md5': 'c7ce8f1f87bec05b3de07fdeafe21a0a', 
 135                 'uploader': 'Vladimir Gavrin', 
 138                 'upload_date': '20120730', 
 141             'skip': 'This video has been removed from public access.', 
 145             # please update if you find a video whose URL follows the same pattern 
 146             'url': 'http://vk.com/video-8871596_164049491', 
 147             'md5': 'a590bcaf3d543576c9bd162812387666', 
 148             'note': 'Only available for registered users', 
 152                 'uploader': 'Триллеры', 
 153                 'title': '► Бойцовский клуб / Fight Club 1999 [HD 720]', 
 155                 'upload_date': '20121218', 
 158             'skip': 'Requires vk account credentials', 
 161             'url': 'http://vk.com/hd_kino_mania?z=video-43215063_168067957%2F15c66b9b533119788d', 
 162             'md5': '4d7a5ef8cf114dfa09577e57b2993202', 
 166                 'uploader': 'Киномания - лучшее из мира кино', 
 169                 'upload_date': '20140328', 
 171             'skip': 'Requires vk account credentials', 
 174             'url': 'http://m.vk.com/video-43215063_169084319?list=125c627d1aa1cebb83&from=wall-43215063_2566540', 
 175             'md5': '0c45586baa71b7cb1d0784ee3f4e00a6', 
 176             'note': 'ivi.ru embed', 
 180                 'title': 'Книга Илая', 
 182                 'upload_date': '20140626', 
 185             'skip': 'Only works from Russia', 
 188             # video (removed?) only available with list id 
 189             'url': 'https://vk.com/video30481095_171201961?list=8764ae2d21f14088d4', 
 190             'md5': '091287af5402239a1051c37ec7b92913', 
 194                 'title': 'ТюменцевВВ_09.07.2015', 
 195                 'uploader': 'Anton Ivanov', 
 197                 'upload_date': '20150709', 
 204             'url': 'https://vk.com/video276849682_170681728', 
 208                 'title': "DSWD Awards 'Children's Joy Foundation, Inc.' Certificate of Registration and License to Operate", 
 209                 'description': 'md5:d9903938abdc74c738af77f527ca0596', 
 211                 'upload_date': '20130116', 
 212                 'uploader': "Children's Joy Foundation", 
 213                 'uploader_id': 'thecjf', 
 219             'url': 'https://vk.com/video-37468416_456239855', 
 221                 'id': 'k3lz2cmXyRuJQSjGHUv', 
 223                 'title': 'md5:d52606645c20b0ddbb21655adaa4f56f', 
 224                 'description': 'md5:c651358f03c56f1150b555c26d90a0fd', 
 225                 'uploader': 'AniLibria.Tv', 
 226                 'upload_date': '20160914', 
 227                 'uploader_id': 'x1p5vl5', 
 228                 'timestamp': 1473877246, 
 231                 'skip_download': True, 
 235             # video key is extra_data not url\d+ 
 236             'url': 'http://vk.com/video-110305615_171782105', 
 237             'md5': 'e13fcda136f99764872e739d13fac1d1', 
 241                 'title': 'S-Dance, репетиции к The way show', 
 242                 'uploader': 'THE WAY SHOW | 17 апреля', 
 243                 'timestamp': 1454870100, 
 244                 'upload_date': '20160207', 
 249             # finished live stream, postlive_mp4 
 250             'url': 'https://vk.com/videos-387766?z=video-387766_456242764%2Fpl_-387766_-2', 
 251             'md5': '90d22d051fccbbe9becfccc615be6791', 
 255                 'title': 'ИгроМир 2016 — день 1', 
 256                 'uploader': 'Игромания', 
 262             # live stream, hls and rtmp links, most likely already finished live 
 263             # stream by the time you are reading this comment 
 264             'url': 'https://vk.com/video-140332_456239111', 
 265             'only_matching': True, 
 268             # removed video, just testing that we match the pattern 
 269             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a', 
 270             'only_matching': True, 
 273             # age restricted video, requires vk account credentials 
 274             'url': 'https://vk.com/video205387401_164765225', 
 275             'only_matching': True, 
 279             'url': 'https://vk.com/video-76116461_171554880', 
 280             'only_matching': True, 
 283             'url': 'http://new.vk.com/video205387401_165548505', 
 284             'only_matching': True, 
 287             # This video is no longer available, because its author has been blocked. 
 288             'url': 'https://vk.com/video-10639516_456240611', 
 289             'only_matching': True, 
 293     def _real_extract(self
, url
): 
 294         mobj 
= re
.match(self
._VALID
_URL
, url
) 
 295         video_id 
= mobj
.group('videoid') 
 298             info_url 
= 'https://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
 
 299             # Some videos (removed?) can only be downloaded with list id specified 
 300             list_id 
= mobj
.group('list_id') 
 302                 info_url 
+= '&list=%s' % list_id
 
 304             info_url 
= 'http://vk.com/video_ext.php?' + mobj
.group('embed_query') 
 305             video_id 
= '%s_%s' % (mobj
.group('oid'), mobj
.group('id')) 
 307         info_page 
= self
._download
_webpage
(info_url
, video_id
) 
 309         error_message 
= self
._html
_search
_regex
( 
 310             [r
'(?s)<!><div[^>]+class="video_layer_message"[^>]*>(.+?)</div>', 
 311                 r
'(?s)<div[^>]+id="video_ext_msg"[^>]*>(.+?)</div>'], 
 312             info_page
, 'error message', default
=None) 
 314             raise ExtractorError(error_message
, expected
=True) 
 316         if re
.search(r
'<!>/login\.php\?.*\bact=security_check', info_page
): 
 317             raise ExtractorError( 
 318                 'You are trying to log in from an unusual location. You should confirm ownership at vk.com to log in with this IP.', 
 321         ERROR_COPYRIGHT 
= 'Video %s has been removed from public access due to rightholder complaint.' 
 324             r
'>Видеозапись .*? была изъята из публичного доступа в связи с обращением правообладателя.<': 
 327             r
'>The video .*? was removed from public access by request of the copyright holder.<': 
 330             r
'<!>Please log in or <': 
 331             'Video %s is only available for registered users, ' 
 332             'use --username and --password options to provide account credentials.', 
 335             'Video %s does not exist.', 
 337             r
'<!>Видео временно недоступно': 
 338             'Video %s is temporarily unavailable.', 
 341             'Access denied to video %s.', 
 343             r
'<!>Видеозапись недоступна, так как её автор был заблокирован.': 
 344             'Video %s is no longer available, because its author has been blocked.', 
 346             r
'<!>This video is no longer available, because its author has been blocked.': 
 347             'Video %s is no longer available, because its author has been blocked.', 
 350         for error_re
, error_msg 
in ERRORS
.items(): 
 351             if re
.search(error_re
, info_page
): 
 352                 raise ExtractorError(error_msg 
% video_id
, expected
=True) 
 354         youtube_url 
= YoutubeIE
._extract
_url
(info_page
) 
 356             return self
.url_result(youtube_url
, ie
=YoutubeIE
.ie_key()) 
 358         vimeo_url 
= VimeoIE
._extract
_url
(url
, info_page
) 
 359         if vimeo_url 
is not None: 
 360             return self
.url_result(vimeo_url
) 
 362         pladform_url 
= PladformIE
._extract
_url
(info_page
) 
 364             return self
.url_result(pladform_url
) 
 366         m_rutube 
= re
.search( 
 367             r
'\ssrc="((?:https?:)?//rutube\.ru\\?/(?:video|play)\\?/embed(?:.*?))\\?"', info_page
) 
 368         if m_rutube 
is not None: 
 369             rutube_url 
= self
._proto
_relative
_url
( 
 370                 m_rutube
.group(1).replace('\\', '')) 
 371             return self
.url_result(rutube_url
) 
 373         dailymotion_urls 
= DailymotionIE
._extract
_urls
(info_page
) 
 375             return self
.url_result(dailymotion_urls
[0], DailymotionIE
.ie_key()) 
 377         m_opts 
= re
.search(r
'(?s)var\s+opts\s*=\s*({.+?});', info_page
) 
 379             m_opts_url 
= re
.search(r
"url\s*:\s*'((?!/\b)[^']+)", m_opts
.group(1)) 
 381                 opts_url 
= m_opts_url
.group(1) 
 382                 if opts_url
.startswith('//'): 
 383                     opts_url 
= 'http:' + opts_url
 
 384                 return self
.url_result(opts_url
) 
 386         # vars does not look to be served anymore since 24.10.2016 
 387         data 
= self
._parse
_json
( 
 389                 r
'var\s+vars\s*=\s*({.+?});', info_page
, 'vars', default
='{}'), 
 390             video_id
, fatal
=False) 
 392         # <!json> is served instead 
 394             data 
= self
._parse
_json
( 
 396                     r
'<!json>\s*({.+?})\s*<!>', info_page
, 'json', default
='{}'), 
 399                 data 
= data
['player']['params'][0] 
 402             data 
= self
._parse
_json
( 
 404                     r
'var\s+playerParams\s*=\s*({.+?})\s*;\s*\n', info_page
, 
 406                 video_id
)['params'][0] 
 408         title 
= unescapeHTML(data
['md_title']) 
 411         # 3 = post live (finished live) 
 412         is_live 
= data
.get('live') == 2 
 414             title 
= self
._live
_title
(title
) 
 416         timestamp 
= unified_timestamp(self
._html
_search
_regex
( 
 417             r
'class=["\']mv_info_date
[^
>]+>([^
<]+)(?
:<|
from)', info_page, 
 418             'upload date
', fatal=False)) 
 420         view_count = str_to_int(self._search_regex( 
 421             r'class=["\']mv_views_count[^>]+>\s*([\d,.]+)', 
 422             info_page, 'view count', default=None)) 
 425         for format_id, format_url in data.items(): 
 426             if not isinstance(format_url, compat_str) or not format_url.startswith(('http', '//', 'rtmp')): 
 428             if (format_id.startswith(('url', 'cache')) or 
 429                     format_id in ('extra_data', 'live_mp4', 'postlive_mp4')): 
 430                 height = int_or_none(self._search_regex( 
 431                     r'^(?:url|cache)(\d+)', format_id, 'height', default=None)) 
 433                     'format_id': format_id, 
 437             elif format_id == 'hls': 
 438                 formats.extend(self._extract_m3u8_formats( 
 439                     format_url, video_id, 'mp4', 'm3u8_native', 
 440                     m3u8_id=format_id, fatal=False, live=is_live)) 
 441             elif format_id == 'rtmp': 
 443                     'format_id': format_id, 
 447         self._sort_formats(formats) 
 450             'id': compat_str(data.get('vid') or video_id), 
 453             'thumbnail': data.get('jpg'), 
 454             'uploader': data.get('md_author'), 
 455             'duration': data.get('duration'), 
 456             'timestamp': timestamp, 
 457             'view_count': view_count, 
 462 class VKUserVideosIE(VKBaseIE): 
 463     IE_NAME = 'vk:uservideos' 
 464     IE_DESC = "VK 
- User
's Videos" 
 465     _VALID_URL = r'https?
://(?
:(?
:m|new
)\
.)?vk\
.com
/videos(?P
<id>-?
[0-9]+)(?
!\?.*\bz
=video
)(?
:[/?
#&]|$)' 
 466     _TEMPLATE_URL 
= 'https://vk.com/videos' 
 468         'url': 'http://vk.com/videos205387401', 
 471             'title': "Tom Cruise's Videos", 
 473         'playlist_mincount': 4, 
 475         'url': 'http://vk.com/videos-77521', 
 476         'only_matching': True, 
 478         'url': 'http://vk.com/videos-97664626?section=all', 
 479         'only_matching': True, 
 481         'url': 'http://m.vk.com/videos205387401', 
 482         'only_matching': True, 
 484         'url': 'http://new.vk.com/videos205387401', 
 485         'only_matching': True, 
 488     def _real_extract(self
, url
): 
 489         page_id 
= self
._match
_id
(url
) 
 491         webpage 
= self
._download
_webpage
(url
, page_id
) 
 495                 'http://vk.com/video' + video_id
, 'VK', video_id
=video_id
) 
 496             for video_id 
in orderedSet(re
.findall(r
'href="/video(-?[0-9_]+)"', webpage
))] 
 498         title 
= unescapeHTML(self
._search
_regex
( 
 499             r
'<title>\s*([^<]+?)\s+\|\s+\d+\s+videos', 
 500             webpage
, 'title', default
=page_id
)) 
 502         return self
.playlist_result(entries
, page_id
, title
) 
 505 class VKWallPostIE(VKBaseIE
): 
 506     IE_NAME 
= 'vk:wallpost' 
 507     _VALID_URL 
= r
'https?://(?:(?:(?:(?:m|new)\.)?vk\.com/(?:[^?]+\?.*\bw=)?wall(?P<id>-?\d+_\d+)))' 
 509         # public page URL, audio playlist 
 510         'url': 'https://vk.com/bs.official?w=wall-23538238_35', 
 513             'title': 'Black Shadow - Wall post 23538238_35', 
 514             'description': 'md5:3f84b9c4f9ef499731cf1ced9998cc0c', 
 517             'md5': '5ba93864ec5b85f7ce19a9af4af080f6', 
 519                 'id': '135220665_111806521', 
 521                 'title': 'Black Shadow - Слепое Верование', 
 523                 'uploader': 'Black Shadow', 
 524                 'artist': 'Black Shadow', 
 525                 'track': 'Слепое Верование', 
 528             'md5': '4cc7e804579122b17ea95af7834c9233', 
 530                 'id': '135220665_111802303', 
 532                 'title': 'Black Shadow - Война - Негасимое Бездны Пламя!', 
 534                 'uploader': 'Black Shadow', 
 535                 'artist': 'Black Shadow', 
 536                 'track': 'Война - Негасимое Бездны Пламя!', 
 539                 'skip_download': True, 
 545         'skip': 'Requires vk account credentials', 
 547         # single YouTube embed, no leading - 
 548         'url': 'https://vk.com/wall85155021_6319', 
 550             'id': '85155021_6319', 
 551             'title': 'Sergey Gorbunov - Wall post 85155021_6319', 
 557         'skip': 'Requires vk account credentials', 
 560         'url': 'https://vk.com/wall-23538238_35', 
 561         'only_matching': True, 
 563         # mobile wall page URL 
 564         'url': 'https://m.vk.com/wall-23538238_35', 
 565         'only_matching': True, 
 568     def _real_extract(self
, url
): 
 569         post_id 
= self
._match
_id
(url
) 
 571         wall_url 
= 'https://vk.com/wall%s' % post_id
 
 573         post_id 
= remove_start(post_id
, '-') 
 575         webpage 
= self
._download
_webpage
(wall_url
, post_id
) 
 577         error 
= self
._html
_search
_regex
( 
 578             r
'>Error</div>\s*<div[^>]+class=["\']body
["\'][^>]*>([^<]+)', 
 579             webpage, 'error', default=None) 
 581             raise ExtractorError('VK said: %s' % error, expected=True) 
 583         description = clean_html(get_element_by_class('wall_post_text', webpage)) 
 584         uploader = clean_html(get_element_by_class('author', webpage)) 
 585         thumbnail = self._og_search_thumbnail(webpage) 
 589         audio_ids = re.findall(r'data-full-id=["\'](\d
+_\d
+)', webpage) 
 591             al_audio = self._download_webpage( 
 592                 'https
://vk
.com
/al_audio
.php
', post_id, 
 593                 note='Downloading audio info
', fatal=False, 
 594                 data=urlencode_postdata({ 
 595                     'act
': 'reload_audio
', 
 597                     'ids
': ','.join(audio_ids) 
 600                 Audio = collections.namedtuple( 
 601                     'Audio
', ['id', 'user_id
', 'url
', 'track
', 'artist
', 'duration
']) 
 602                 audios = self._parse_json( 
 604                         r'<!json
>(.+?
)<!>', al_audio, 'audios
', default='[]'), 
 605                     post_id, fatal=False, transform_source=unescapeHTML) 
 606                 if isinstance(audios, list): 
 608                         a = Audio._make(audio[:6]) 
 610                             'id': '%s_%s' % (a.user_id, a.id), 
 612                             'title
': '%s - %s' % (a.artist, a.track) if a.artist and a.track else a.id, 
 613                             'thumbnail
': thumbnail, 
 614                             'duration
': a.duration, 
 615                             'uploader
': uploader, 
 620         for video in re.finditer( 
 621                 r'<a
[^
>]+href
=(["\'])(?P<url>/video(?:-?[\d_]+).*?)\1', webpage): 
 622             entries.append(self.url_result( 
 623                 compat_urlparse.urljoin(url, video.group('url')), VKIE.ie_key())) 
 625         title = 'Wall post %s' % post_id 
 627         return self.playlist_result( 
 628             orderedSet(entries), post_id, 
 629             '%s - %s' % (uploader, title) if uploader else title,