Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/odnoklassniki.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 from .common import InfoExtractor
   5 from ..compat import (
   6     compat_etree_fromstring,
   7     compat_parse_qs,
   8     compat_urllib_parse_unquote,
   9     compat_urllib_parse_urlparse,
  10 )
  11 from ..utils import (
  12     ExtractorError,
  13     unified_strdate,
  14     int_or_none,
  15     qualities,
  16     unescapeHTML,
  17     urlencode_postdata,
  18 )
  19
  20
  21 class OdnoklassnikiIE(InfoExtractor):
  22     _VALID_URL = r'https?://(?:(?:www|m|mobile)\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer|live)/(?P<id>[\d-]+)'
  23     _TESTS = [{
  24         # metadata in JSON
  25         'url': 'http://ok.ru/video/20079905452',
  26         'md5': '0b62089b479e06681abaaca9d204f152',
  27         'info_dict': {
  28             'id': '20079905452',
  29             'ext': 'mp4',
  30             'title': 'Культура меняет нас (прекрасный ролик!))',
  31             'duration': 100,
  32             'upload_date': '20141207',
  33             'uploader_id': '330537914540',
  34             'uploader': 'Виталий Добровольский',
  35             'like_count': int,
  36             'age_limit': 0,
  37         },
  38     }, {
  39         # metadataUrl
  40         'url': 'http://ok.ru/video/63567059965189-0?fromTime=5',
  41         'md5': '6ff470ea2dd51d5d18c295a355b0b6bc',
  42         'info_dict': {
  43             'id': '63567059965189-0',
  44             'ext': 'mp4',
  45             'title': 'Девушка без комплексов ...',
  46             'duration': 191,
  47             'upload_date': '20150518',
  48             'uploader_id': '534380003155',
  49             'uploader': '☭ Андрей Мещанинов ☭',
  50             'like_count': int,
  51             'age_limit': 0,
  52             'start_time': 5,
  53         },
  54     }, {
  55         # YouTube embed (metadataUrl, provider == USER_YOUTUBE)
  56         'url': 'http://ok.ru/video/64211978996595-1',
  57         'md5': '2f206894ffb5dbfcce2c5a14b909eea5',
  58         'info_dict': {
  59             'id': 'V_VztHT5BzY',
  60             'ext': 'mp4',
  61             'title': 'Космическая среда от 26 августа 2015',
  62             'description': 'md5:848eb8b85e5e3471a3a803dae1343ed0',
  63             'duration': 440,
  64             'upload_date': '20150826',
  65             'uploader_id': 'tvroscosmos',
  66             'uploader': 'Телестудия Роскосмоса',
  67             'age_limit': 0,
  68         },
  69     }, {
  70         # YouTube embed (metadata, provider == USER_YOUTUBE, no metadata.movie.title field)
  71         'url': 'http://ok.ru/video/62036049272859-0',
  72         'info_dict': {
  73             'id': '62036049272859-0',
  74             'ext': 'mp4',
  75             'title': 'МУЗЫКА     ДОЖДЯ .',
  76             'description': 'md5:6f1867132bd96e33bf53eda1091e8ed0',
  77             'upload_date': '20120106',
  78             'uploader_id': '473534735899',
  79             'uploader': 'МARINA D',
  80             'age_limit': 0,
  81         },
  82         'params': {
  83             'skip_download': True,
  84         },
  85         'skip': 'Video has not been found',
  86     }, {
  87         'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
  88         'only_matching': True,
  89     }, {
  90         'url': 'http://www.ok.ru/video/20648036891',
  91         'only_matching': True,
  92     }, {
  93         'url': 'http://www.ok.ru/videoembed/20648036891',
  94         'only_matching': True,
  95     }, {
  96         'url': 'http://m.ok.ru/video/20079905452',
  97         'only_matching': True,
  98     }, {
  99         'url': 'http://mobile.ok.ru/video/20079905452',
 100         'only_matching': True,
 101     }, {
 102         'url': 'https://www.ok.ru/live/484531969818',
 103         'only_matching': True,
 104     }]
 105
 106     def _real_extract(self, url):
 107         start_time = int_or_none(compat_parse_qs(
 108             compat_urllib_parse_urlparse(url).query).get('fromTime', [None])[0])
 109
 110         video_id = self._match_id(url)
 111
 112         webpage = self._download_webpage(
 113             'http://ok.ru/video/%s' % video_id, video_id)
 114
 115         error = self._search_regex(
 116             r'[^>]+class="vp_video_stub_txt"[^>]*>([^<]+)<',
 117             webpage, 'error', default=None)
 118         if error:
 119             raise ExtractorError(error, expected=True)
 120
 121         player = self._parse_json(
 122             unescapeHTML(self._search_regex(
 123                 r'data-options=(?P<quote>["\'])(?P<player>{.+?%s.+?})(?P=quote)' % video_id,
 124                 webpage, 'player', group='player')),
 125             video_id)
 126
 127         flashvars = player['flashvars']
 128
 129         metadata = flashvars.get('metadata')
 130         if metadata:
 131             metadata = self._parse_json(metadata, video_id)
 132         else:
 133             data = {}
 134             st_location = flashvars.get('location')
 135             if st_location:
 136                 data['st.location'] = st_location
 137             metadata = self._download_json(
 138                 compat_urllib_parse_unquote(flashvars['metadataUrl']),
 139                 video_id, 'Downloading metadata JSON',
 140                 data=urlencode_postdata(data))
 141
 142         movie = metadata['movie']
 143
 144         # Some embedded videos may not contain title in movie dict (e.g.
 145         # http://ok.ru/video/62036049272859-0) thus we allow missing title
 146         # here and it's going to be extracted later by an extractor that
 147         # will process the actual embed.
 148         provider = metadata.get('provider')
 149         title = movie['title'] if provider == 'UPLOADED_ODKL' else movie.get('title')
 150
 151         thumbnail = movie.get('poster')
 152         duration = int_or_none(movie.get('duration'))
 153
 154         author = metadata.get('author', {})
 155         uploader_id = author.get('id')
 156         uploader = author.get('name')
 157
 158         upload_date = unified_strdate(self._html_search_meta(
 159             'ya:ovs:upload_date', webpage, 'upload date', default=None))
 160
 161         age_limit = None
 162         adult = self._html_search_meta(
 163             'ya:ovs:adult', webpage, 'age limit', default=None)
 164         if adult:
 165             age_limit = 18 if adult == 'true' else 0
 166
 167         like_count = int_or_none(metadata.get('likeCount'))
 168
 169         info = {
 170             'id': video_id,
 171             'title': title,
 172             'thumbnail': thumbnail,
 173             'duration': duration,
 174             'upload_date': upload_date,
 175             'uploader': uploader,
 176             'uploader_id': uploader_id,
 177             'like_count': like_count,
 178             'age_limit': age_limit,
 179             'start_time': start_time,
 180         }
 181
 182         if provider == 'USER_YOUTUBE':
 183             info.update({
 184                 '_type': 'url_transparent',
 185                 'url': movie['contentId'],
 186             })
 187             return info
 188
 189         assert title
 190         if provider == 'LIVE_TV_APP':
 191             info['title'] = self._live_title(title)
 192
 193         quality = qualities(('4', '0', '1', '2', '3', '5'))
 194
 195         formats = [{
 196             'url': f['url'],
 197             'ext': 'mp4',
 198             'format_id': f['name'],
 199         } for f in metadata['videos']]
 200
 201         m3u8_url = metadata.get('hlsManifestUrl')
 202         if m3u8_url:
 203             formats.extend(self._extract_m3u8_formats(
 204                 m3u8_url, video_id, 'mp4', 'm3u8_native',
 205                 m3u8_id='hls', fatal=False))
 206
 207         dash_manifest = metadata.get('metadataEmbedded')
 208         if dash_manifest:
 209             formats.extend(self._parse_mpd_formats(
 210                 compat_etree_fromstring(dash_manifest), 'mpd'))
 211
 212         for fmt in formats:
 213             fmt_type = self._search_regex(
 214                 r'\btype[/=](\d)', fmt['url'],
 215                 'format type', default=None)
 216             if fmt_type:
 217                 fmt['quality'] = quality(fmt_type)
 218
 219         # Live formats
 220         m3u8_url = metadata.get('hlsMasterPlaylistUrl')
 221         if m3u8_url:
 222             formats.extend(self._extract_m3u8_formats(
 223                 m3u8_url, video_id, 'mp4', entry_protocol='m3u8',
 224                 m3u8_id='hls', fatal=False))
 225         rtmp_url = metadata.get('rtmpUrl')
 226         if rtmp_url:
 227             formats.append({
 228                 'url': rtmp_url,
 229                 'format_id': 'rtmp',
 230                 'ext': 'flv',
 231             })
 232
 233         self._sort_formats(formats)
 234
 235         info['formats'] = formats
 236         return info