Merge pull request #4 from debian-janitor/lintian-fixes

[youtubedl] / youtube_dl / extractor / vlive.py

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index b9718901b8339e3d5fee85a15d20514b038b66aa..f79531e6f3a2e922b0369706cc0d76a22feb2499 100644 (file)
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -6,24 +6,21 @@ import time
 import itertools
 
 from .common import InfoExtractor
 import itertools
 
 from .common import InfoExtractor

-from ..compat import (
-    compat_urllib_parse_urlencode,
-    compat_str,
-)
+from .naver import NaverBaseIE
+from ..compat import compat_str

 from ..utils import (
 from ..utils import (

-    dict_get,

     ExtractorError,
     ExtractorError,

-    float_or_none,
-    int_or_none,
+    merge_dicts,

     remove_start,
     try_get,
     urlencode_postdata,
 )
 
 
     remove_start,
     try_get,
     urlencode_postdata,
 )
 
 

-class VLiveIE(InfoExtractor):
+class VLiveIE(NaverBaseIE):

     IE_NAME = 'vlive'
     _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'
     IE_NAME = 'vlive'
     _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<id>[0-9]+)'

+    _NETRC_MACHINE = 'vlive'

     _TESTS = [{
         'url': 'http://www.vlive.tv/video/1326',
         'md5': 'cc7314812855ce56de70a06a27314983',
     _TESTS = [{
         'url': 'http://www.vlive.tv/video/1326',
         'md5': 'cc7314812855ce56de70a06a27314983',


@@ -33,6 +30,7 @@ class VLiveIE(InfoExtractor):
             'title': "[V LIVE] Girl's Day's Broadcast",
             'creator': "Girl's Day",
             'view_count': int,
             'title': "[V LIVE] Girl's Day's Broadcast",
             'creator': "Girl's Day",
             'view_count': int,

+            'uploader_id': 'muploader_a',

         },
     }, {
         'url': 'http://www.vlive.tv/video/16937',
         },
     }, {
         'url': 'http://www.vlive.tv/video/16937',


@@ -43,17 +41,65 @@ class VLiveIE(InfoExtractor):
             'creator': 'EXO',
             'view_count': int,
             'subtitles': 'mincount:12',
             'creator': 'EXO',
             'view_count': int,
             'subtitles': 'mincount:12',

+            'uploader_id': 'muploader_j',

         },
         'params': {
             'skip_download': True,
         },
         },
         'params': {
             'skip_download': True,
         },

+    }, {
+        'url': 'https://www.vlive.tv/video/129100',
+        'md5': 'ca2569453b79d66e5b919e5d308bff6b',
+        'info_dict': {
+            'id': '129100',
+            'ext': 'mp4',
+            'title': '[V LIVE] [BTS+] Run BTS! 2019 - EP.71 :: Behind the scene',
+            'creator': 'BTS+',
+            'view_count': int,
+            'subtitles': 'mincount:10',
+        },
+        'skip': 'This video is only available for CH+ subscribers',

     }]
 
     }]
 

+    @classmethod
+    def suitable(cls, url):
+        return False if VLivePlaylistIE.suitable(url) else super(VLiveIE, cls).suitable(url)
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        email, password = self._get_login_info()
+        if None in (email, password):
+            return
+
+        def is_logged_in():
+            login_info = self._download_json(
+                'https://www.vlive.tv/auth/loginInfo', None,
+                note='Downloading login info',
+                headers={'Referer': 'https://www.vlive.tv/home'})
+            return try_get(
+                login_info, lambda x: x['message']['login'], bool) or False
+
+        LOGIN_URL = 'https://www.vlive.tv/auth/email/login'
+        self._request_webpage(
+            LOGIN_URL, None, note='Downloading login cookies')
+
+        self._download_webpage(
+            LOGIN_URL, None, note='Logging in',
+            data=urlencode_postdata({'email': email, 'pwd': password}),
+            headers={
+                'Referer': LOGIN_URL,
+                'Content-Type': 'application/x-www-form-urlencoded'
+            })
+
+        if not is_logged_in():
+            raise ExtractorError('Unable to log in', expected=True)
+

     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(

-            'http://www.vlive.tv/video/%s' % video_id, video_id)
+            'https://www.vlive.tv/video/%s' % video_id, video_id)

 
         VIDEO_PARAMS_RE = r'\bvlive\.video\.init\(([^)]+)'
         VIDEO_PARAMS_FIELD = 'video params'
 
         VIDEO_PARAMS_RE = r'\bvlive\.video\.init\(([^)]+)'
         VIDEO_PARAMS_FIELD = 'video params'


@@ -70,13 +116,10 @@ class VLiveIE(InfoExtractor):
         status, long_video_id, key = params[2], params[5], params[6]
         status = remove_start(status, 'PRODUCT_')
 
         status, long_video_id, key = params[2], params[5], params[6]
         status = remove_start(status, 'PRODUCT_')
 

-        if status == 'LIVE_ON_AIR' or status == 'BIG_EVENT_ON_AIR':
+        if status in ('LIVE_ON_AIR', 'BIG_EVENT_ON_AIR'):

             return self._live(video_id, webpage)
             return self._live(video_id, webpage)

-        elif status == 'VOD_ON_AIR' or status == 'BIG_EVENT_INTRO':
-            if long_video_id and key:
-                return self._replay(video_id, webpage, long_video_id, key)
-            else:
-                status = 'COMING_SOON'
+        elif status in ('VOD_ON_AIR', 'BIG_EVENT_INTRO'):
+            return self._replay(video_id, webpage, long_video_id, key)

 
         if status == 'LIVE_END':
             raise ExtractorError('Uploading for replay. Please wait...',
 
         if status == 'LIVE_END':
             raise ExtractorError('Uploading for replay. Please wait...',


@@ -87,13 +130,15 @@ class VLiveIE(InfoExtractor):
             raise ExtractorError('We are sorry, '
                                  'but the live broadcast has been canceled.',
                                  expected=True)
             raise ExtractorError('We are sorry, '
                                  'but the live broadcast has been canceled.',
                                  expected=True)

+        elif status == 'ONLY_APP':
+            raise ExtractorError('Unsupported video type', expected=True)

         else:
             raise ExtractorError('Unknown status %s' % status)
 
     def _get_common_fields(self, webpage):
         title = self._og_search_title(webpage)
         creator = self._html_search_regex(
         else:
             raise ExtractorError('Unknown status %s' % status)
 
     def _get_common_fields(self, webpage):
         title = self._og_search_title(webpage)
         creator = self._html_search_regex(

-            r'<div[^>]+class="info_area"[^>]*>\s*<a\s+[^>]*>([^<]+)',
+            r'<div[^>]+class="info_area"[^>]*>\s*(?:<em[^>]*>.*?</em\s*>\s*)?<a\s+[^>]*>([^<]+)',

             webpage, 'creator', fatal=False)
         thumbnail = self._og_search_thumbnail(webpage)
         return {
             webpage, 'creator', fatal=False)
         thumbnail = self._og_search_thumbnail(webpage)
         return {


@@ -103,14 +148,7 @@ class VLiveIE(InfoExtractor):
         }
 
     def _live(self, video_id, webpage):
         }
 
     def _live(self, video_id, webpage):

-        init_page = self._download_webpage(
-            'http://www.vlive.tv/video/init/view',
-            video_id, note='Downloading live webpage',
-            data=urlencode_postdata({'videoSeq': video_id}),
-            headers={
-                'Referer': 'http://www.vlive.tv/video/%s' % video_id,
-                'Content-Type': 'application/x-www-form-urlencoded'
-            })
+        init_page = self._download_init_page(video_id)

 
         live_params = self._search_regex(
             r'"liveStreamInfo"\s*:\s*(".*"),',
 
         live_params = self._search_regex(
             r'"liveStreamInfo"\s*:\s*(".*"),',


@@ -136,45 +174,30 @@ class VLiveIE(InfoExtractor):
         return info
 
     def _replay(self, video_id, webpage, long_video_id, key):
         return info
 
     def _replay(self, video_id, webpage, long_video_id, key):

-        playinfo = self._download_json(
-            'http://global.apis.naver.com/rmcnmv/rmcnmv/vod_play_videoInfo.json?%s'
-            % compat_urllib_parse_urlencode({
-                'videoId': long_video_id,
-                'key': key,
-                'ptc': 'http',
-                'doct': 'json',  # document type (xml or json)
-                'cpt': 'vtt',  # captions type (vtt or ttml)
-            }), video_id)
-
-        formats = [{
-            'url': vid['source'],
-            'format_id': vid.get('encodingOption', {}).get('name'),
-            'abr': float_or_none(vid.get('bitrate', {}).get('audio')),
-            'vbr': float_or_none(vid.get('bitrate', {}).get('video')),
-            'width': int_or_none(vid.get('encodingOption', {}).get('width')),
-            'height': int_or_none(vid.get('encodingOption', {}).get('height')),
-            'filesize': int_or_none(vid.get('size')),
-        } for vid in playinfo.get('videos', {}).get('list', []) if vid.get('source')]
-        self._sort_formats(formats)
-
-        view_count = int_or_none(playinfo.get('meta', {}).get('count'))
-
-        subtitles = {}
-        for caption in playinfo.get('captions', {}).get('list', []):
-            lang = dict_get(caption, ('locale', 'language', 'country', 'label'))
-            if lang and caption.get('source'):
-                subtitles[lang] = [{
-                    'ext': 'vtt',
-                    'url': caption['source']}]
-
-        info = self._get_common_fields(webpage)
-        info.update({
-            'id': video_id,
-            'formats': formats,
-            'view_count': view_count,
-            'subtitles': subtitles,
-        })
-        return info
+        if '' in (long_video_id, key):
+            init_page = self._download_init_page(video_id)
+            video_info = self._parse_json(self._search_regex(
+                (r'(?s)oVideoStatus\s*=\s*({.+?})\s*</script',
+                 r'(?s)oVideoStatus\s*=\s*({.+})'), init_page, 'video info'),
+                video_id)
+            if video_info.get('status') == 'NEED_CHANNEL_PLUS':
+                self.raise_login_required(
+                    'This video is only available for CH+ subscribers')
+            long_video_id, key = video_info['vid'], video_info['inkey']
+
+        return merge_dicts(
+            self._get_common_fields(webpage),
+            self._extract_video_info(video_id, long_video_id, key))
+
+    def _download_init_page(self, video_id):
+        return self._download_webpage(
+            'https://www.vlive.tv/video/init/view',
+            video_id, note='Downloading live webpage',
+            data=urlencode_postdata({'videoSeq': video_id}),
+            headers={
+                'Referer': 'https://www.vlive.tv/video/%s' % video_id,
+                'Content-Type': 'application/x-www-form-urlencoded'
+            })

 
 
 class VLiveChannelIE(InfoExtractor):
 
 
 class VLiveChannelIE(InfoExtractor):


@@ -232,7 +255,12 @@ class VLiveChannelIE(InfoExtractor):
                 query={
                     'app_id': app_id,
                     'channelSeq': channel_seq,
                 query={
                     'app_id': app_id,
                     'channelSeq': channel_seq,

-                    'maxNumOfRows': 1000,
+                    # Large values of maxNumOfRows (~300 or above) may cause
+                    # empty responses (see [1]), e.g. this happens for [2] that
+                    # has more than 300 videos.
+                    # 1. https://github.com/ytdl-org/youtube-dl/issues/13830
+                    # 2. http://channels.vlive.tv/EDBF.
+                    'maxNumOfRows': 100,

                     '_': int(time.time()),
                     'pageNo': page_num
                 }
                     '_': int(time.time()),
                     'pageNo': page_num
                 }


@@ -261,3 +289,79 @@ class VLiveChannelIE(InfoExtractor):
 
         return self.playlist_result(
             entries, channel_code, channel_name)
 
         return self.playlist_result(
             entries, channel_code, channel_name)

+
+
+class VLivePlaylistIE(InfoExtractor):
+    IE_NAME = 'vlive:playlist'
+    _VALID_URL = r'https?://(?:(?:www|m)\.)?vlive\.tv/video/(?P<video_id>[0-9]+)/playlist/(?P<id>[0-9]+)'
+    _VIDEO_URL_TEMPLATE = 'http://www.vlive.tv/video/%s'
+    _TESTS = [{
+        # regular working playlist
+        'url': 'https://www.vlive.tv/video/117956/playlist/117963',
+        'info_dict': {
+            'id': '117963',
+            'title': '아이돌룸(IDOL ROOM) 41회 - (여자)아이들'
+        },
+        'playlist_mincount': 10
+    }, {
+        # playlist with no playlistVideoSeqs
+        'url': 'http://www.vlive.tv/video/22867/playlist/22912',
+        'info_dict': {
+            'id': '22867',
+            'ext': 'mp4',
+            'title': '[V LIVE] Valentine Day Message from MINA',
+            'creator': 'TWICE',
+            'view_count': int
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }]
+
+    def _build_video_result(self, video_id, message):
+        self.to_screen(message)
+        return self.url_result(
+            self._VIDEO_URL_TEMPLATE % video_id,
+            ie=VLiveIE.ie_key(), video_id=video_id)
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id, playlist_id = mobj.group('video_id', 'id')
+
+        if self._downloader.params.get('noplaylist'):
+            return self._build_video_result(
+                video_id,
+                'Downloading just video %s because of --no-playlist'
+                % video_id)
+
+        self.to_screen(
+            'Downloading playlist %s - add --no-playlist to just download video'
+            % playlist_id)
+
+        webpage = self._download_webpage(
+            'http://www.vlive.tv/video/%s/playlist/%s'
+            % (video_id, playlist_id), playlist_id)
+
+        raw_item_ids = self._search_regex(
+            r'playlistVideoSeqs\s*=\s*(\[[^]]+\])', webpage,
+            'playlist video seqs', default=None, fatal=False)
+
+        if not raw_item_ids:
+            return self._build_video_result(
+                video_id,
+                'Downloading just video %s because no playlist was found'
+                % video_id)
+
+        item_ids = self._parse_json(raw_item_ids, playlist_id)
+
+        entries = [
+            self.url_result(
+                self._VIDEO_URL_TEMPLATE % item_id, ie=VLiveIE.ie_key(),
+                video_id=compat_str(item_id))
+            for item_id in item_ids]
+
+        playlist_name = self._html_search_regex(
+            r'<div[^>]+class="[^"]*multicam_playlist[^>]*>\s*<h3[^>]+>([^<]+)',
+            webpage, 'playlist title', fatal=False)
+
+        return self.playlist_result(entries, playlist_id, playlist_name)