X-Git-Url: https://git.rapsys.eu/youtubedl/blobdiff_plain/3477c644417600d9ec8f8d2a44f82da0a4b15eb5..d1d8237037fb87a57bd1d073c3b9e642d4a75016:/youtube_dl/extractor/bilibili.py?ds=inline diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py index 6c66a12..b17047b 100644 --- a/youtube_dl/extractor/bilibili.py +++ b/youtube_dl/extractor/bilibili.py @@ -1,144 +1,229 @@ # coding: utf-8 from __future__ import unicode_literals +import calendar +import datetime import re -import itertools -import json from .common import InfoExtractor from ..compat import ( compat_etree_fromstring, + compat_str, + compat_parse_qs, + compat_xml_parse_error, ) from ..utils import ( - int_or_none, - unified_strdate, ExtractorError, + int_or_none, + float_or_none, + xpath_text, ) class BiliBiliIE(InfoExtractor): - _VALID_URL = r'http://www\.bilibili\.(?:tv|com)/video/av(?P<id>[0-9]+)/' + _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)' _TESTS = [{ 'url': 'http://www.bilibili.tv/video/av1074402/', - 'md5': '2c301e4dab317596e837c3e7633e7d86', + 'md5': '5f7d29e1a2872f3df0cf76b1f87d3788', 'info_dict': { - 'id': '1074402_part1', + 'id': '1554319', 'ext': 'flv', 'title': 'ãéå·åãé泡沫', - 'duration': 308, + 'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923', + 'duration': 308.067, + 'timestamp': 1398012660, 'upload_date': '20140420', 'thumbnail': 're:^https?://.+\.jpg', + 'uploader': 'èåæ¡', + 'uploader_id': '156160', }, }, { 'url': 'http://www.bilibili.com/video/av1041170/', 'info_dict': { 'id': '1041170', 'title': 'ãBD1080Pãåè¯ã诸ç¥&å¼åã', + 'description': 'è¿æ¯ä¸ªç¥å¥çæ äº~æ¯ä¸ªäººä¸çå¼¹å¹ä¸ç»èµ°å¦~åå©å¦ï¼~', }, 'playlist_count': 9, + }, { + 'url': 'http://www.bilibili.com/video/av4808130/', + 'info_dict': { + 'id': '4808130', + 'title': 'ãé¿ç¯ãåå¦A梦443ãééã', + 'description': '(2016.05.27)æ¥ç»å客人çè¸å§&amp;寻æ¯å åéé æ±æï¼åè½®å°å¨æ¥ä¸çç°å¨æå°å®¶ å°é¢www.pixiv.net/member_illust.php?mode=medium&amp;illust_id=56912929', + }, + 'playlist': [{ + 'md5': '55cdadedf3254caaa0d5d27cf20a8f9c', + 'info_dict': { + 'id': '4808130_part1', + 'ext': 'flv', + 'title': 'ãé¿ç¯ãåå¦A梦443ãééã', + 'description': '(2016.05.27)æ¥ç»å客人çè¸å§&amp;寻æ¯å åéé æ±æï¼åè½®å°å¨æ¥ä¸çç°å¨æå°å®¶ å°é¢www.pixiv.net/member_illust.php?mode=medium&amp;illust_id=56912929', + 'timestamp': 1464564180, + 'upload_date': '20160529', + 'uploader': 'å欢æé¢', + 'uploader_id': '151066', + }, + }, { + 'md5': '926f9f67d0c482091872fbd8eca7ea3d', + 'info_dict': { + 'id': '4808130_part2', + 'ext': 'flv', + 'title': 'ãé¿ç¯ãåå¦A梦443ãééã', + 'description': '(2016.05.27)æ¥ç»å客人çè¸å§&amp;寻æ¯å åéé æ±æï¼åè½®å°å¨æ¥ä¸çç°å¨æå°å®¶ å°é¢www.pixiv.net/member_illust.php?mode=medium&amp;illust_id=56912929', + 'timestamp': 1464564180, + 'upload_date': '20160529', + 'uploader': 'å欢æé¢', + 'uploader_id': '151066', + }, + }, { + 'md5': '4b7b225b968402d7c32348c646f1fd83', + 'info_dict': { + 'id': '4808130_part3', + 'ext': 'flv', + 'title': 'ãé¿ç¯ãåå¦A梦443ãééã', + 'description': '(2016.05.27)æ¥ç»å客人çè¸å§&amp;寻æ¯å åéé æ±æï¼åè½®å°å¨æ¥ä¸çç°å¨æå°å®¶ å°é¢www.pixiv.net/member_illust.php?mode=medium&amp;illust_id=56912929', + 'timestamp': 1464564180, + 'upload_date': '20160529', + 'uploader': 'å欢æé¢', + 'uploader_id': '151066', + }, + }, { + 'md5': '7b795e214166501e9141139eea236e91', + 'info_dict': { + 'id': '4808130_part4', + 'ext': 'flv', + 'title': 'ãé¿ç¯ãåå¦A梦443ãééã', + 'description': '(2016.05.27)æ¥ç»å客人çè¸å§&amp;寻æ¯å åéé æ±æï¼åè½®å°å¨æ¥ä¸çç°å¨æå°å®¶ å°é¢www.pixiv.net/member_illust.php?mode=medium&amp;illust_id=56912929', + 'timestamp': 1464564180, + 'upload_date': '20160529', + 'uploader': 'å欢æé¢', + 'uploader_id': '151066', + }, + }], + }, { + # Missing upload time + 'url': 'http://www.bilibili.com/video/av1867637/', + 'info_dict': { + 'id': '2880301', + 'ext': 'flv', + 'title': 'ãHDTVããåå§ãå²³ç¶å²³æ¯çé¾å½ ï¼2014ï¼ãæ³å½ç¥¨æ¿å åã', + 'description': 'ä¸ä¸ªä¿¡å¥å¤©ä¸»æçæ³å½æ§å¼ä¼ ç»èµäº§é¶çº§å®¶åºä¸æå个女å¿ãä¸ä¸ªå¥³å¿å´åå«æ¾äºé¿æ伯ãç¹å¤ªãä¸å½ä¸å¤«ï¼è夫è妻å¯ç¬æç¼å©ä¸æªå«çå°å¥³å¿è½æ¾ä¸ä¸ªä¿¡å¥å¤©ä¸»æçæ³å½ç½äººï¼ç»æ没æ³å°å°å¥³å¿æ¾äºä¸ä½éè£é»äººâ¦â¦ãè¿æ¬¡åºè¯¥ä¸ä¼è·³å¸§äºã', + 'uploader': 'é»å¤ä¸ºç«', + 'uploader_id': '610729', + }, + 'params': { + # Just to test metadata extraction + 'skip_download': True, + }, + 'expected_warnings': ['upload time'], }] + # BiliBili blocks keys from time to time. The current key is extracted from + # the Android client + # TODO: find the sign algorithm used in the flash player + _APP_KEY = '86385cdc024c0f6c' + def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') - if '(æ¤è§é¢ä¸åå¨æ被å é¤)' in webpage: - raise ExtractorError( - 'The video does not exist or was deleted', expected=True) + webpage = self._download_webpage(url, video_id) - if '>ä½ æ²¡ææéæµè§ï¼ ç±äºçæç¸å ³é®é¢ æ们ä¸å¯¹æ¨æå¨çå°åºæä¾æå¡<' in webpage: - raise ExtractorError( - 'The video is not available in your region due to copyright reasons', - expected=True) + params = compat_parse_qs(self._search_regex( + [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)', + r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'], + webpage, 'player parameters')) + cid = params['cid'][0] - video_code = self._search_regex( - r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code') + info_xml_str = self._download_webpage( + 'http://interface.bilibili.com/v_cdn_play', + cid, query={'appkey': self._APP_KEY, 'cid': cid}, + note='Downloading video info page') - title = self._html_search_meta( - 'media:title', video_code, 'title', fatal=True) - duration_str = self._html_search_meta( - 'duration', video_code, 'duration') - if duration_str is None: - duration = None + err_msg = None + durls = None + info_xml = None + try: + info_xml = compat_etree_fromstring(info_xml_str.encode('utf-8')) + except compat_xml_parse_error: + info_json = self._parse_json(info_xml_str, video_id, fatal=False) + err_msg = (info_json or {}).get('error_text') else: - duration_mobj = re.match( - r'^T(?:(?P<hours>[0-9]+)H)?(?P<minutes>[0-9]+)M(?P<seconds>[0-9]+)S$', - duration_str) - duration = ( - int_or_none(duration_mobj.group('hours'), default=0) * 3600 + - int(duration_mobj.group('minutes')) * 60 + - int(duration_mobj.group('seconds'))) - upload_date = unified_strdate(self._html_search_meta( - 'uploadDate', video_code, fatal=False)) - thumbnail = self._html_search_meta( - 'thumbnailUrl', video_code, 'thumbnail', fatal=False) - - cid = self._search_regex(r'cid=(\d+)', webpage, 'cid') + err_msg = xpath_text(info_xml, './message') - entries = [] + if info_xml is not None: + durls = info_xml.findall('./durl') + if not durls: + if err_msg: + raise ExtractorError('%s said: %s' % (self.IE_NAME, err_msg), expected=True) + else: + raise ExtractorError('No videos found!') - lq_page = self._download_webpage( - 'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid, - video_id, - note='Downloading LQ video info' - ) - try: - err_info = json.loads(lq_page) - raise ExtractorError( - 'BiliBili said: ' + err_info['error_text'], expected=True) - except ValueError: - pass - - lq_doc = compat_etree_fromstring(lq_page) - lq_durls = lq_doc.findall('./durl') - - hq_doc = self._download_xml( - 'http://interface.bilibili.com/playurl?appkey=1&cid=%s' % cid, - video_id, - note='Downloading HQ video info', - fatal=False, - ) - if hq_doc is not False: - hq_durls = hq_doc.findall('./durl') - assert len(lq_durls) == len(hq_durls) - else: - hq_durls = itertools.repeat(None) + entries = [] - i = 1 - for lq_durl, hq_durl in zip(lq_durls, hq_durls): + for durl in durls: + size = xpath_text(durl, ['./filesize', './size']) formats = [{ - 'format_id': 'lq', - 'quality': 1, - 'url': lq_durl.find('./url').text, - 'filesize': int_or_none( - lq_durl.find('./size'), get_attr='text'), + 'url': durl.find('./url').text, + 'filesize': int_or_none(size), }] - if hq_durl is not None: + for backup_url in durl.findall('./backup_url/url'): formats.append({ - 'format_id': 'hq', - 'quality': 2, - 'ext': 'flv', - 'url': hq_durl.find('./url').text, - 'filesize': int_or_none( - hq_durl.find('./size'), get_attr='text'), + 'url': backup_url.text, + # backup URLs have lower priorities + 'preference': -2 if 'hd.mp4' in backup_url.text else -3, }) + self._sort_formats(formats) entries.append({ - 'id': '%s_part%d' % (video_id, i), - 'title': title, + 'id': '%s_part%s' % (cid, xpath_text(durl, './order')), + 'duration': int_or_none(xpath_text(durl, './length'), 1000), 'formats': formats, - 'duration': duration, - 'upload_date': upload_date, - 'thumbnail': thumbnail, }) - i += 1 - - return { - '_type': 'multi_video', - 'entries': entries, - 'id': video_id, - 'title': title + title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title') + description = self._html_search_meta('description', webpage) + datetime_str = self._html_search_regex( + r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False) + timestamp = None + if datetime_str: + timestamp = calendar.timegm(datetime.datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M').timetuple()) + + # TODO 'view_count' requires deobfuscating Javascript + info = { + 'id': compat_str(cid), + 'title': title, + 'description': description, + 'timestamp': timestamp, + 'thumbnail': self._html_search_meta('thumbnailUrl', webpage), + 'duration': float_or_none(xpath_text(info_xml, './timelength'), scale=1000), } + + uploader_mobj = re.search( + r'<a[^>]+href="https?://space\.bilibili\.com/(?P<id>\d+)"[^>]+title="(?P<name>[^"]+)"', + webpage) + if uploader_mobj: + info.update({ + 'uploader': uploader_mobj.group('name'), + 'uploader_id': uploader_mobj.group('id'), + }) + + for entry in entries: + entry.update(info) + + if len(entries) == 1: + return entries[0] + else: + for idx, entry in enumerate(entries): + entry['id'] = '%s_part%d' % (video_id, (idx + 1)) + + return { + '_type': 'multi_video', + 'id': video_id, + 'title': title, + 'description': description, + 'entries': entries, + }