X-Git-Url: https://git.rapsys.eu/youtubedl/blobdiff_plain/e8cd8c4bd832446f1971215b9fedc4531555dc1a..f4289d8c83b2ef4f79c675e01198056ae9a2aea9:/youtube_dl/extractor/imdb.py?ds=sidebyside diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py index f40769e..a313019 100644 --- a/youtube_dl/extractor/imdb.py +++ b/youtube_dl/extractor/imdb.py @@ -1,89 +1,147 @@ from __future__ import unicode_literals -import re +import base64 import json +import re from .common import InfoExtractor from ..utils import ( - compat_urlparse, - get_element_by_attribute, + determine_ext, + mimetype2ext, + parse_duration, + qualities, + try_get, + url_or_none, ) class ImdbIE(InfoExtractor): IE_NAME = 'imdb' IE_DESC = 'Internet Movie Database trailers' - _VALID_URL = r'http://(?:www|m)\.imdb\.com/video/imdb/vi(?P\d+)' + _VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video|title|list).*?[/-]vi(?P\d+)' - _TEST = { + _TESTS = [{ 'url': 'http://www.imdb.com/video/imdb/vi2524815897', - 'md5': '9f34fa777ade3a6e57a054fdbcb3a068', 'info_dict': { 'id': '2524815897', 'ext': 'mp4', - 'title': 'Ice Age: Continental Drift Trailer (No. 2) - IMDb', - 'description': 'md5:9061c2219254e5d14e03c25c98e96a81', + 'title': 'No. 2', + 'description': 'md5:87bd0bdc61e351f21f20d2d7441cb4e7', + 'duration': 152, } - } + }, { + 'url': 'http://www.imdb.com/video/_/vi2524815897', + 'only_matching': True, + }, { + 'url': 'http://www.imdb.com/title/tt1667889/?ref_=ext_shr_eml_vi#lb-vi2524815897', + 'only_matching': True, + }, { + 'url': 'http://www.imdb.com/title/tt1667889/#lb-vi2524815897', + 'only_matching': True, + }, { + 'url': 'http://www.imdb.com/videoplayer/vi1562949145', + 'only_matching': True, + }, { + 'url': 'http://www.imdb.com/title/tt4218696/videoplayer/vi2608641561', + 'only_matching': True, + }, { + 'url': 'https://www.imdb.com/list/ls009921623/videoplayer/vi260482329', + 'only_matching': True, + }] def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - webpage = self._download_webpage('http://www.imdb.com/video/imdb/vi%s' % video_id, video_id) - descr = get_element_by_attribute('itemprop', 'description', webpage) - available_formats = re.findall( - r'case \'(?P.*?)\' :$\s+url = \'(?P.*?)\'', webpage, - flags=re.MULTILINE) + video_id = self._match_id(url) + + data = self._download_json( + 'https://www.imdb.com/ve/data/VIDEO_PLAYBACK_DATA', video_id, + query={ + 'key': base64.b64encode(json.dumps({ + 'type': 'VIDEO_PLAYER', + 'subType': 'FORCE_LEGACY', + 'id': 'vi%s' % video_id, + }).encode()).decode(), + })[0] + + quality = qualities(('SD', '480p', '720p', '1080p')) formats = [] - for f_id, f_path in available_formats: - f_path = f_path.strip() - format_page = self._download_webpage( - compat_urlparse.urljoin(url, f_path), - 'Downloading info for %s format' % f_id) - json_data = self._search_regex( - r']+class="imdb-player-data"[^>]*?>(.*?)', - format_page, 'json data', flags=re.DOTALL) - info = json.loads(json_data) - format_info = info['videoPlayerObject']['video'] + for encoding in data['videoLegacyEncodings']: + if not encoding or not isinstance(encoding, dict): + continue + video_url = url_or_none(encoding.get('url')) + if not video_url: + continue + ext = mimetype2ext(encoding.get( + 'mimeType')) or determine_ext(video_url) + if ext == 'm3u8': + formats.extend(self._extract_m3u8_formats( + video_url, video_id, 'mp4', entry_protocol='m3u8_native', + preference=1, m3u8_id='hls', fatal=False)) + continue + format_id = encoding.get('definition') formats.append({ - 'format_id': f_id, - 'url': format_info['url'], + 'format_id': format_id, + 'url': video_url, + 'ext': ext, + 'quality': quality(format_id), }) + self._sort_formats(formats) + + webpage = self._download_webpage( + 'https://www.imdb.com/video/vi' + video_id, video_id) + video_metadata = self._parse_json(self._search_regex( + r'args\.push\(\s*({.+?})\s*\)\s*;', webpage, + 'video metadata'), video_id) + + video_info = video_metadata.get('VIDEO_INFO') + if video_info and isinstance(video_info, dict): + info = try_get( + video_info, lambda x: x[list(video_info.keys())[0]][0], dict) + else: + info = {} + + title = self._html_search_meta( + ['og:title', 'twitter:title'], webpage) or self._html_search_regex( + r'(.+?)', webpage, 'title', + default=None) or info['videoTitle'] return { 'id': video_id, - 'title': self._og_search_title(webpage), + 'title': title, + 'alt_title': info.get('videoSubTitle'), 'formats': formats, - 'description': descr, - 'thumbnail': format_info['slate'], + 'description': info.get('videoDescription'), + 'thumbnail': url_or_none(try_get( + video_metadata, lambda x: x['videoSlate']['source'])), + 'duration': parse_duration(info.get('videoRuntime')), } class ImdbListIE(InfoExtractor): IE_NAME = 'imdb:list' IE_DESC = 'Internet Movie Database lists' - _VALID_URL = r'http://www\.imdb\.com/list/(?P[\da-zA-Z_-]{11})' - + _VALID_URL = r'https?://(?:www\.)?imdb\.com/list/ls(?P\d{9})(?!/videoplayer/vi\d+)' + _TEST = { + 'url': 'https://www.imdb.com/list/ls009921623/', + 'info_dict': { + 'id': '009921623', + 'title': 'The Bourne Legacy', + 'description': 'A list of trailers, clips, and more from The Bourne Legacy, starring Jeremy Renner and Rachel Weisz.', + }, + 'playlist_count': 8, + } + def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - list_id = mobj.group('id') - - # RSS XML is sometimes malformed - rss = self._download_webpage('http://rss.imdb.com/list/%s' % list_id, list_id, 'Downloading list RSS') - list_title = self._html_search_regex(r'(.*?)', rss, 'list title') - - # Export is independent of actual author_id, but returns 404 if no author_id is provided. - # However, passing dummy author_id seems to be enough. - csv = self._download_webpage('http://www.imdb.com/list/export?list_id=%s&author_id=ur00000000' % list_id, - list_id, 'Downloading list CSV') - - entries = [] - for item in csv.split('\n')[1:]: - cols = item.split(',') - if len(cols) < 2: - continue - item_id = cols[1][1:-1] - if item_id.startswith('vi'): - entries.append(self.url_result('http://www.imdb.com/video/imdb/%s' % item_id, 'Imdb')) - - return self.playlist_result(entries, list_id, list_title) + list_id = self._match_id(url) + webpage = self._download_webpage(url, list_id) + entries = [ + self.url_result('http://www.imdb.com' + m, 'Imdb') + for m in re.findall(r'href="(/list/ls%s/videoplayer/vi[^"]+)"' % list_id, webpage)] + + list_title = self._html_search_regex( + r']+class="[^"]*header[^"]*"[^>]*>(.*?)', + webpage, 'list title') + list_description = self._html_search_regex( + r']+class="[^"]*list-description[^"]*"[^>]*>

(.*?)

', + webpage, 'list description') + + return self.playlist_result(entries, list_id, list_title, list_description)