Merge tag 'upstream/2017.05.18.1'

[youtubedl] / youtube_dl / extractor / bilibili.py

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index b17047b399b6630fe2334aa24f0a5e97aed8506f..1e3f25515d3596f7b196a16ab867b7b5e8c891ab 100644 (file)
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -1,209 +1,158 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
 # coding: utf-8
 from __future__ import unicode_literals
 

-import calendar
-import datetime
+import hashlib

 import re
 
 from .common import InfoExtractor
 from ..compat import (
 import re
 
 from .common import InfoExtractor
 from ..compat import (

-    compat_etree_fromstring,
-    compat_str,

     compat_parse_qs,
     compat_parse_qs,

-    compat_xml_parse_error,
+    compat_urlparse,

 )
 from ..utils import (
     ExtractorError,
     int_or_none,
     float_or_none,
 )
 from ..utils import (
     ExtractorError,
     int_or_none,
     float_or_none,

-    xpath_text,
+    parse_iso8601,
+    smuggle_url,
+    strip_jsonp,
+    unified_timestamp,
+    unsmuggle_url,
+    urlencode_postdata,

 )
 
 
 class BiliBiliIE(InfoExtractor):
 )
 
 
 class BiliBiliIE(InfoExtractor):

-    _VALID_URL = r'https?://www\.bilibili\.(?:tv|com)/video/av(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.|bangumi\.|)bilibili\.(?:tv|com)/(?:video/av|anime/(?P<anime_id>\d+)/play#)(?P<id>\d+)'

 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',
 
     _TESTS = [{
         'url': 'http://www.bilibili.tv/video/av1074402/',

-        'md5': '5f7d29e1a2872f3df0cf76b1f87d3788',
+        'md5': '9fa226fe2b8a9a4d5a69b4c6a183417e',

         'info_dict': {
         'info_dict': {

-            'id': '1554319',
-            'ext': 'flv',
+            'id': '1074402',
+            'ext': 'mp4',

             'title': '【金坷垃】金泡沫',
             'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
             'title': '【金坷垃】金泡沫',
             'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',

-            'duration': 308.067,
+            'duration': 308.315,

             'timestamp': 1398012660,
             'upload_date': '20140420',
             'timestamp': 1398012660,
             'upload_date': '20140420',

-            'thumbnail': 're:^https?://.+\.jpg',
+            'thumbnail': r're:^https?://.+\.jpg',

             'uploader': '菊子桑',
             'uploader_id': '156160',
         },
     }, {
             'uploader': '菊子桑',
             'uploader_id': '156160',
         },
     }, {

-        'url': 'http://www.bilibili.com/video/av1041170/',
-        'info_dict': {
-            'id': '1041170',
-            'title': '【BD1080P】刀语【诸神&异域】',
-            'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
-        },
-        'playlist_count': 9,
-    }, {
-        'url': 'http://www.bilibili.com/video/av4808130/',
-        'info_dict': {
-            'id': '4808130',
-            'title': '【长篇】哆啦A梦443【钉铛】',
-            'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-        },
-        'playlist': [{
-            'md5': '55cdadedf3254caaa0d5d27cf20a8f9c',
-            'info_dict': {
-                'id': '4808130_part1',
-                'ext': 'flv',
-                'title': '【长篇】哆啦A梦443【钉铛】',
-                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-                'timestamp': 1464564180,
-                'upload_date': '20160529',
-                'uploader': '喜欢拉面',
-                'uploader_id': '151066',
-            },
-        }, {
-            'md5': '926f9f67d0c482091872fbd8eca7ea3d',
-            'info_dict': {
-                'id': '4808130_part2',
-                'ext': 'flv',
-                'title': '【长篇】哆啦A梦443【钉铛】',
-                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-                'timestamp': 1464564180,
-                'upload_date': '20160529',
-                'uploader': '喜欢拉面',
-                'uploader_id': '151066',
-            },
-        }, {
-            'md5': '4b7b225b968402d7c32348c646f1fd83',
-            'info_dict': {
-                'id': '4808130_part3',
-                'ext': 'flv',
-                'title': '【长篇】哆啦A梦443【钉铛】',
-                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-                'timestamp': 1464564180,
-                'upload_date': '20160529',
-                'uploader': '喜欢拉面',
-                'uploader_id': '151066',
-            },
-        }, {
-            'md5': '7b795e214166501e9141139eea236e91',
-            'info_dict': {
-                'id': '4808130_part4',
-                'ext': 'flv',
-                'title': '【长篇】哆啦A梦443【钉铛】',
-                'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
-                'timestamp': 1464564180,
-                'upload_date': '20160529',
-                'uploader': '喜欢拉面',
-                'uploader_id': '151066',
-            },
-        }],
+        # Tested in BiliBiliBangumiIE
+        'url': 'http://bangumi.bilibili.com/anime/1869/play#40062',
+        'only_matching': True,

     }, {
     }, {

-        # Missing upload time
-        'url': 'http://www.bilibili.com/video/av1867637/',
+        'url': 'http://bangumi.bilibili.com/anime/5802/play#100643',
+        'md5': '3f721ad1e75030cc06faf73587cfec57',

         'info_dict': {
         'info_dict': {

-            'id': '2880301',
-            'ext': 'flv',
-            'title': '【HDTV】【喜剧】岳父岳母真难当 （2014）【法国票房冠军】',
-            'description': '一个信奉天主教的法国旧式传统资产阶级家庭中有四个女儿。三个女儿却分别找了阿拉伯、犹太、中国丈夫，老夫老妻唯独期盼剩下未嫁的小女儿能找一个信奉天主教的法国白人，结果没想到小女儿找了一位非裔黑人……【这次应该不会跳帧了】',
-            'uploader': '黑夜为猫',
-            'uploader_id': '610729',
+            'id': '100643',
+            'ext': 'mp4',
+            'title': 'CHAOS;CHILD',
+            'description': '如果你是神明，并且能够让妄想成为现实。那你会进行怎么样的妄想？是淫靡的世界？独裁社会？毁灭性的制裁？还是……2015年，涩谷。从6年前发生的大灾害“涩谷地震”之后复兴了的这个街区里新设立的私立高中...',

         },
         },

-        'params': {
-            # Just to test metadata extraction
-            'skip_download': True,
-        },
-        'expected_warnings': ['upload time'],
+        'skip': 'Geo-restricted to China',

     }]
 
     }]
 

-    # BiliBili blocks keys from time to time. The current key is extracted from
-    # the Android client
-    # TODO: find the sign algorithm used in the flash player
-    _APP_KEY = '86385cdc024c0f6c'
+    _APP_KEY = '84956560bc028eb7'
+    _BILIBILI_KEY = '94aba54af9065f71de72f5508f1cd42e'
+
+    def _report_error(self, result):
+        if 'message' in result:
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, result['message']), expected=True)
+        elif 'code' in result:
+            raise ExtractorError('%s returns error %d' % (self.IE_NAME, result['code']), expected=True)
+        else:
+            raise ExtractorError('Can\'t extract Bangumi episode ID')

 
     def _real_extract(self, url):
 
     def _real_extract(self, url):

+        url, smuggled_data = unsmuggle_url(url, {})
+

         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')

-
+        anime_id = mobj.group('anime_id')

         webpage = self._download_webpage(url, video_id)
 
         webpage = self._download_webpage(url, video_id)
 

-        params = compat_parse_qs(self._search_regex(
-            [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
-             r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
-            webpage, 'player parameters'))
-        cid = params['cid'][0]
-
-        info_xml_str = self._download_webpage(
-            'http://interface.bilibili.com/v_cdn_play',
-            cid, query={'appkey': self._APP_KEY, 'cid': cid},
-            note='Downloading video info page')
-
-        err_msg = None
-        durls = None
-        info_xml = None
-        try:
-            info_xml = compat_etree_fromstring(info_xml_str.encode('utf-8'))
-        except compat_xml_parse_error:
-            info_json = self._parse_json(info_xml_str, video_id, fatal=False)
-            err_msg = (info_json or {}).get('error_text')
+        if 'anime/' not in url:
+            cid = compat_parse_qs(self._search_regex(
+                [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
+                 r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
+                webpage, 'player parameters'))['cid'][0]

         else:
         else:

-            err_msg = xpath_text(info_xml, './message')
+            if 'no_bangumi_tip' not in smuggled_data:
+                self.to_screen('Downloading episode %s. To download all videos in anime %s, re-run youtube-dl with %s' % (
+                    video_id, anime_id, compat_urlparse.urljoin(url, '//bangumi.bilibili.com/anime/%s' % anime_id)))
+            headers = {
+                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+            }
+            headers.update(self.geo_verification_headers())
+
+            js = self._download_json(
+                'http://bangumi.bilibili.com/web_api/get_source', video_id,
+                data=urlencode_postdata({'episode_id': video_id}),
+                headers=headers)
+            if 'result' not in js:
+                self._report_error(js)
+            cid = js['result']['cid']
+
+        payload = 'appkey=%s&cid=%s&otype=json&quality=2&type=mp4' % (self._APP_KEY, cid)
+        sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest()

 
 

-        if info_xml is not None:
-            durls = info_xml.findall('./durl')
-        if not durls:
-            if err_msg:
-                raise ExtractorError('%s said: %s' % (self.IE_NAME, err_msg), expected=True)
-            else:
-                raise ExtractorError('No videos found!')
+        video_info = self._download_json(
+            'http://interface.bilibili.com/playurl?%s&sign=%s' % (payload, sign),
+            video_id, note='Downloading video info page',
+            headers=self.geo_verification_headers())
+
+        if 'durl' not in video_info:
+            self._report_error(video_info)

 
         entries = []
 
 
         entries = []
 

-        for durl in durls:
-            size = xpath_text(durl, ['./filesize', './size'])
+        for idx, durl in enumerate(video_info['durl']):

             formats = [{
             formats = [{

-                'url': durl.find('./url').text,
-                'filesize': int_or_none(size),
+                'url': durl['url'],
+                'filesize': int_or_none(durl['size']),

             }]
             }]

-            for backup_url in durl.findall('./backup_url/url'):
+            for backup_url in durl.get('backup_url', []):

                 formats.append({
                 formats.append({

-                    'url': backup_url.text,
+                    'url': backup_url,

                     # backup URLs have lower priorities
                     # backup URLs have lower priorities

-                    'preference': -2 if 'hd.mp4' in backup_url.text else -3,
+                    'preference': -2 if 'hd.mp4' in backup_url else -3,
+                })
+
+            for a_format in formats:
+                a_format.setdefault('http_headers', {}).update({
+                    'Referer': url,

                 })
 
             self._sort_formats(formats)
 
             entries.append({
                 })
 
             self._sort_formats(formats)
 
             entries.append({

-                'id': '%s_part%s' % (cid, xpath_text(durl, './order')),
-                'duration': int_or_none(xpath_text(durl, './length'), 1000),
+                'id': '%s_part%s' % (video_id, idx),
+                'duration': float_or_none(durl.get('length'), 1000),

                 'formats': formats,
             })
 
         title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title')
         description = self._html_search_meta('description', webpage)
                 'formats': formats,
             })
 
         title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title')
         description = self._html_search_meta('description', webpage)

-        datetime_str = self._html_search_regex(
-            r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False)
-        timestamp = None
-        if datetime_str:
-            timestamp = calendar.timegm(datetime.datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M').timetuple())
+        timestamp = unified_timestamp(self._html_search_regex(
+            r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', default=None))
+        thumbnail = self._html_search_meta(['og:image', 'thumbnailUrl'], webpage)

 
         # TODO 'view_count' requires deobfuscating Javascript
         info = {
 
         # TODO 'view_count' requires deobfuscating Javascript
         info = {

-            'id': compat_str(cid),
+            'id': video_id,

             'title': title,
             'description': description,
             'timestamp': timestamp,
             'title': title,
             'description': description,
             'timestamp': timestamp,

-            'thumbnail': self._html_search_meta('thumbnailUrl', webpage),
-            'duration': float_or_none(xpath_text(info_xml, './timelength'), scale=1000),
+            'thumbnail': thumbnail,
+            'duration': float_or_none(video_info.get('timelength'), scale=1000),

         }
 
         uploader_mobj = re.search(
         }
 
         uploader_mobj = re.search(

-            r'<a[^>]+href="https?://space\.bilibili\.com/(?P<id>\d+)"[^>]+title="(?P<name>[^"]+)"',
+            r'<a[^>]+href="(?:https?:)?//space\.bilibili\.com/(?P<id>\d+)"[^>]+title="(?P<name>[^"]+)"',

             webpage)
         if uploader_mobj:
             info.update({
             webpage)
         if uploader_mobj:
             info.update({


@@ -227,3 +176,70 @@ class BiliBiliIE(InfoExtractor):
                 'description': description,
                 'entries': entries,
             }
                 'description': description,
                 'entries': entries,
             }

+
+
+class BiliBiliBangumiIE(InfoExtractor):
+    _VALID_URL = r'https?://bangumi\.bilibili\.com/anime/(?P<id>\d+)'
+
+    IE_NAME = 'bangumi.bilibili.com'
+    IE_DESC = 'BiliBili番剧'
+
+    _TESTS = [{
+        'url': 'http://bangumi.bilibili.com/anime/1869',
+        'info_dict': {
+            'id': '1869',
+            'title': '混沌武士',
+            'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
+        },
+        'playlist_count': 26,
+    }, {
+        'url': 'http://bangumi.bilibili.com/anime/1869',
+        'info_dict': {
+            'id': '1869',
+            'title': '混沌武士',
+            'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
+        },
+        'playlist': [{
+            'md5': '91da8621454dd58316851c27c68b0c13',
+            'info_dict': {
+                'id': '40062',
+                'ext': 'mp4',
+                'title': '混沌武士',
+                'description': '故事发生在日本的江户时代。风是一个小酒馆的打工女。一日，酒馆里来了一群恶霸，虽然他们的举动令风十分不满，但是毕竟风只是一届女流，无法对他们采取什么行动，只能在心里嘟哝。这时，酒家里又进来了个“不良份子...',
+                'timestamp': 1414538739,
+                'upload_date': '20141028',
+                'episode': '疾风怒涛 Tempestuous Temperaments',
+                'episode_number': 1,
+            },
+        }],
+        'params': {
+            'playlist_items': '1',
+        },
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if BiliBiliIE.suitable(url) else super(BiliBiliBangumiIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        bangumi_id = self._match_id(url)
+
+        # Sometimes this API returns a JSONP response
+        season_info = self._download_json(
+            'http://bangumi.bilibili.com/jsonp/seasoninfo/%s.ver' % bangumi_id,
+            bangumi_id, transform_source=strip_jsonp)['result']
+
+        entries = [{
+            '_type': 'url_transparent',
+            'url': smuggle_url(episode['webplay_url'], {'no_bangumi_tip': 1}),
+            'ie_key': BiliBiliIE.ie_key(),
+            'timestamp': parse_iso8601(episode.get('update_time'), delimiter=' '),
+            'episode': episode.get('index_title'),
+            'episode_number': int_or_none(episode.get('index')),
+        } for episode in season_info['episodes']]
+
+        entries = sorted(entries, key=lambda entry: entry.get('episode_number'))
+
+        return self.playlist_result(
+            entries, bangumi_id,
+            season_info.get('bangumi_title'), season_info.get('evaluate'))