Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/odnoklassniki.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 from .common import InfoExtractor
   5 from ..compat import (
   6     compat_etree_fromstring,
   7     compat_parse_qs,
   8     compat_urllib_parse_unquote,
   9     compat_urllib_parse_urlparse,
  10 )
  11 from ..utils import (
  12     ExtractorError,
  13     unified_strdate,
  14     int_or_none,
  15     qualities,
  16     unescapeHTML,
  17     urlencode_postdata,
  18 )
  19
  20
  21 class OdnoklassnikiIE(InfoExtractor):
  22     _VALID_URL = r'''(?x)
  23                 https?://
  24                     (?:(?:www|m|mobile)\.)?
  25                     (?:odnoklassniki|ok)\.ru/
  26                     (?:
  27                         video(?:embed)?/|
  28                         web-api/video/moviePlayer/|
  29                         live/|
  30                         dk\?.*?st\.mvId=
  31                     )
  32                     (?P<id>[\d-]+)
  33                 '''
  34     _TESTS = [{
  35         # metadata in JSON
  36         'url': 'http://ok.ru/video/20079905452',
  37         'md5': '0b62089b479e06681abaaca9d204f152',
  38         'info_dict': {
  39             'id': '20079905452',
  40             'ext': 'mp4',
  41             'title': 'Культура меняет нас (прекрасный ролик!))',
  42             'duration': 100,
  43             'upload_date': '20141207',
  44             'uploader_id': '330537914540',
  45             'uploader': 'Виталий Добровольский',
  46             'like_count': int,
  47             'age_limit': 0,
  48         },
  49     }, {
  50         # metadataUrl
  51         'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
  52         'md5': '6ff470ea2dd51d5d18c295a355b0b6bc',
  53         'info_dict': {
  54             'id': '63567059965189-0',
  55             'ext': 'mp4',
  56             'title': 'Девушка без комплексов ...',
  57             'duration': 191,
  58             'upload_date': '20150518',
  59             'uploader_id': '534380003155',
  60             'uploader': '☭ Андрей Мещанинов ☭',
  61             'like_count': int,
  62             'age_limit': 0,
  63             'start_time': 5,
  64         },
  65     }, {
  66         # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
  67         'url': 'http://ok.ru/video/64211978996595-1',
  68         'md5': '2f206894ffb5dbfcce2c5a14b909eea5',
  69         'info_dict': {
  70             'id': 'V_VztHT5BzY',
  71             'ext': 'mp4',
  72             'title': 'Космическая среда от 26 августа 2015',
  73             'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
  74             'duration': 440,
  75             'upload_date': '20150826',
  76             'uploader_id': 'tvroscosmos',
  77             'uploader': 'Телестудия Роскосмоса',
  78             'age_limit': 0,
  79         },
  80     }, {
  81         # YouTube embed (metadata, provider == USER_YOUTUBE, no metadata.movie.title field)
  82         'url': 'http://ok.ru/video/62036049272859-0',
  83         'info_dict': {
  84             'id': '62036049272859-0',
  85             'ext': 'mp4',
  86             'title': 'МУЗЫКА     ДОЖДЯ .',
  87             'description': 'md5:6f1867132bd96e33bf53eda1091e8ed0',
  88             'upload_date': '20120106',
  89             'uploader_id': '473534735899',
  90             'uploader': 'МARINA D',
  91             'age_limit': 0,
  92         },
  93         'params': {
  94             'skip_download': True,
  95         },
  96         'skip': 'Video has not been found',
  97     }, {
  98         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
  99         'only_matching': True,
 100     }, {
 101         'url': 'http://www.ok.ru/video/20648036891',
 102         'only_matching': True,
 103     }, {
 104         'url': 'http://www.ok.ru/videoembed/20648036891',
 105         'only_matching': True,
 106     }, {
 107         'url': 'http://m.ok.ru/video/20079905452',
 108         'only_matching': True,
 109     }, {
 110         'url': 'http://mobile.ok.ru/video/20079905452',
 111         'only_matching': True,
 112     }, {
 113         'url': 'https://www.ok.ru/live/484531969818',
 114         'only_matching': True,
 115     }, {
 116         'url': 'https://m.ok.ru/dk?st.cmd=movieLayer&st.discId=863789452017&st.retLoc=friend&st.rtu=%2Fdk%3Fst.cmd%3DfriendMovies%26st.mode%3Down%26st.mrkId%3D%257B%2522uploadedMovieMarker%2522%253A%257B%2522marker%2522%253A%25221519410114503%2522%252C%2522hasMore%2522%253Atrue%257D%252C%2522sharedMovieMarker%2522%253A%257B%2522marker%2522%253Anull%252C%2522hasMore%2522%253Afalse%257D%257D%26st.friendId%3D561722190321%26st.frwd%3Don%26_prevCmd%3DfriendMovies%26tkn%3D7257&st.discType=MOVIE&st.mvId=863789452017&_prevCmd=friendMovies&tkn=3648#lst#',
 117         'only_matching': True,
 118     }, {
 119         # Paid video
 120         'url': 'https://ok.ru/video/954886983203',
 121         'only_matching': True,
 122     }]
 123
 124     def _real_extract(self, url):
 125         start_time = int_or_none(compat_parse_qs(
 126             compat_urllib_parse_urlparse(url).query).get('fromTime', [None])[0])
 127
 128         video_id = self._match_id(url)
 129
 130         webpage = self._download_webpage(
 131             'http://ok.ru/video/%s' % video_id, video_id)
 132
 133         error = self._search_regex(
 134             r'[^>]+class="vp_video_stub_txt"[^>]*>([^<]+)<',
 135             webpage, 'error', default=None)
 136         if error:
 137             raise ExtractorError(error, expected=True)
 138
 139         player = self._parse_json(
 140             unescapeHTML(self._search_regex(
 141                 r'data-options=(?P<quote>["\'])(?P<player>{.+?%s.+?})(?P=quote)' % video_id,
 142                 webpage, 'player', group='player')),
 143             video_id)
 144
 145         flashvars = player['flashvars']
 146
 147         metadata = flashvars.get('metadata')
 148         if metadata:
 149             metadata = self._parse_json(metadata, video_id)
 150         else:
 151             data = {}
 152             st_location = flashvars.get('location')
 153             if st_location:
 154                 data['st.location'] = st_location
 155             metadata = self._download_json(
 156                 compat_urllib_parse_unquote(flashvars['metadataUrl']),
 157                 video_id, 'Downloading metadata JSON',
 158                 data=urlencode_postdata(data))
 159
 160         movie = metadata['movie']
 161
 162         # Some embedded videos may not contain title in movie dict (e.g.
 163         # http://ok.ru/video/62036049272859-0) thus we allow missing title
 164         # here and it's going to be extracted later by an extractor that
 165         # will process the actual embed.
 166         provider = metadata.get('provider')
 167         title = movie['title'] if provider == 'UPLOADED_ODKL' else movie.get('title')
 168
 169         thumbnail = movie.get('poster')
 170         duration = int_or_none(movie.get('duration'))
 171
 172         author = metadata.get('author', {})
 173         uploader_id = author.get('id')
 174         uploader = author.get('name')
 175
 176         upload_date = unified_strdate(self._html_search_meta(
 177             'ya:ovs:upload_date', webpage, 'upload date', default=None))
 178
 179         age_limit = None
 180         adult = self._html_search_meta(
 181             'ya:ovs:adult', webpage, 'age limit', default=None)
 182         if adult:
 183             age_limit = 18 if adult == 'true' else 0
 184
 185         like_count = int_or_none(metadata.get('likeCount'))
 186
 187         info = {
 188             'id': video_id,
 189             'title': title,
 190             'thumbnail': thumbnail,
 191             'duration': duration,
 192             'upload_date': upload_date,
 193             'uploader': uploader,
 194             'uploader_id': uploader_id,
 195             'like_count': like_count,
 196             'age_limit': age_limit,
 197             'start_time': start_time,
 198         }
 199
 200         if provider == 'USER_YOUTUBE':
 201             info.update({
 202                 '_type': 'url_transparent',
 203                 'url': movie['contentId'],
 204             })
 205             return info
 206
 207         assert title
 208         if provider == 'LIVE_TV_APP':
 209             info['title'] = self._live_title(title)
 210
 211         quality = qualities(('4', '0', '1', '2', '3', '5'))
 212
 213         formats = [{
 214             'url': f['url'],
 215             'ext': 'mp4',
 216             'format_id': f['name'],
 217         } for f in metadata['videos']]
 218
 219         m3u8_url = metadata.get('hlsManifestUrl')
 220         if m3u8_url:
 221             formats.extend(self._extract_m3u8_formats(
 222                 m3u8_url, video_id, 'mp4', 'm3u8_native',
 223                 m3u8_id='hls', fatal=False))
 224
 225         dash_manifest = metadata.get('metadataEmbedded')
 226         if dash_manifest:
 227             formats.extend(self._parse_mpd_formats(
 228                 compat_etree_fromstring(dash_manifest), 'mpd'))
 229
 230         for fmt in formats:
 231             fmt_type = self._search_regex(
 232                 r'\btype[/=](\d)', fmt['url'],
 233                 'format type', default=None)
 234             if fmt_type:
 235                 fmt['quality'] = quality(fmt_type)
 236
 237         # Live formats
 238         m3u8_url = metadata.get('hlsMasterPlaylistUrl')
 239         if m3u8_url:
 240             formats.extend(self._extract_m3u8_formats(
 241                 m3u8_url, video_id, 'mp4', entry_protocol='m3u8',
 242                 m3u8_id='hls', fatal=False))
 243         rtmp_url = metadata.get('rtmpUrl')
 244         if rtmp_url:
 245             formats.append({
 246                 'url': rtmp_url,
 247                 'format_id': 'rtmp',
 248                 'ext': 'flv',
 249             })
 250
 251         if not formats:
 252             payment_info = metadata.get('paymentInfo')
 253             if payment_info:
 254                 raise ExtractorError('This video is paid, subscribe to download it', expected=True)
 255
 256         self._sort_formats(formats)
 257
 258         info['formats'] = formats
 259         return info