Prepare to upload

[youtubedl] / youtube_dl / extractor / dramafever.py

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index d836c1a6c27ccd757e4e1f6574e8851c90775193..6b60e542b37418473e1ee2a959cb6ae08a60d2e5 100644 (file)
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -1,27 +1,27 @@
-# encoding: utf-8
+# coding: utf-8

 from __future__ import unicode_literals
 
 import itertools
 
 from __future__ import unicode_literals
 
 import itertools
 

-from .common import InfoExtractor
+from .amp import AMPIE

 from ..compat import (
     compat_HTTPError,
 from ..compat import (
     compat_HTTPError,

-    compat_urllib_parse,

     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
     clean_html,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
     clean_html,

-    determine_ext,

     int_or_none,
     int_or_none,

-    parse_iso8601,
+    remove_end,

     sanitized_Request,
     sanitized_Request,

+    urlencode_postdata

 )
 
 
 )
 
 

-class DramaFeverBaseIE(InfoExtractor):
+class DramaFeverBaseIE(AMPIE):

     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'
     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'

+    _GEO_COUNTRIES = ['US', 'CA']

 
     _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
 
 
     _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
 


@@ -52,14 +52,14 @@ class DramaFeverBaseIE(InfoExtractor):
         }
 
         request = sanitized_Request(
         }
 
         request = sanitized_Request(

-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, urlencode_postdata(login_form))

         response = self._download_webpage(
         response = self._download_webpage(

-            request, None, 'Logging in as %s' % username)
+            request, None, 'Logging in')

 
         if all(logout_pattern not in response
                for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
             error = self._html_search_regex(
 
         if all(logout_pattern not in response
                for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
             error = self._html_search_regex(

-                r'(?s)class="hidden-xs prompt"[^>]*>(.+?)<',
+                r'(?s)<h\d[^>]+\bclass="hidden-xs prompt"[^>]*>(.+?)</h\d',

                 response, 'error message', default=None)
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)
                 response, 'error message', default=None)
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)


@@ -68,71 +68,64 @@ class DramaFeverBaseIE(InfoExtractor):
 
 class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'
 
 class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'

-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
+    _TESTS = [{

         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
             'ext': 'flv',
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
             'ext': 'flv',

-            'title': 'Cooking with Shin 4512.1',
+            'title': 'Cooking with Shin',

             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',

-            'thumbnail': 're:^https?://.*\.jpg',
+            'episode': 'Episode 1',
+            'episode_number': 1,
+            'thumbnail': r're:^https?://.*\.jpg',

             'timestamp': 1404336058,
             'upload_date': '20140702',
             'timestamp': 1404336058,
             'upload_date': '20140702',

-            'duration': 343,
-        }
-    }
+            'duration': 344,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1',
+        'info_dict': {
+            'id': '4826.4',
+            'ext': 'flv',
+            'title': 'Mnet Asian Music Awards 2015',
+            'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
+            'episode': 'Mnet Asian Music Awards 2015 - Part 3',
+            'episode_number': 4,
+            'thumbnail': r're:^https?://.*\.jpg',
+            'timestamp': 1450213200,
+            'upload_date': '20151215',
+            'duration': 5359,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.dramafever.com/zh-cn/drama/4972/15/Doctor_Romantic/',
+        'only_matching': True,
+    }]

 
     def _real_extract(self, url):
         video_id = self._match_id(url).replace('/', '.')
 
         try:
 
     def _real_extract(self, url):
         video_id = self._match_id(url).replace('/', '.')
 
         try:

-            feed = self._download_json(
-                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id,
-                video_id, 'Downloading episode JSON')['channel']['item']
+            info = self._extract_feed_info(
+                'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)

         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):

-                raise ExtractorError(
-                    'Currently unavailable in your country.', expected=True)
+                self.raise_geo_restricted(
+                    msg='Currently unavailable in your country',
+                    countries=self._GEO_COUNTRIES)

             raise
 
             raise
 

-        media_group = feed.get('media-group', {})
-
-        formats = []
-        for media_content in media_group['media-content']:
-            src = media_content.get('@attributes', {}).get('url')
-            if not src:
-                continue
-            ext = determine_ext(src)
-            if ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    src, video_id, f4m_id='hds'))
-            elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    src, video_id, 'mp4', m3u8_id='hls'))
-            else:
-                formats.append({
-                    'url': src,
-                })
-        self._sort_formats(formats)
-
-        title = media_group.get('media-title')
-        description = media_group.get('media-description')
-        duration = int_or_none(media_group['media-content'][0].get('@attributes', {}).get('duration'))
-        thumbnail = self._proto_relative_url(
-            media_group.get('media-thumbnail', {}).get('@attributes', {}).get('url'))
-        timestamp = parse_iso8601(feed.get('pubDate'), ' ')
-
-        subtitles = {}
-        for media_subtitle in media_group.get('media-subTitle', []):
-            lang = media_subtitle.get('@attributes', {}).get('lang')
-            href = media_subtitle.get('@attributes', {}).get('href')
-            if not lang or not href:
-                continue
-            subtitles[lang] = [{
-                'ext': 'ttml',
-                'url': href,
-            }]
+        # title is postfixed with video id for some reason, removing
+        if info.get('title'):
+            info['title'] = remove_end(info['title'], video_id).strip()

 
         series_id, episode_number = video_id.split('.')
         episode_info = self._download_json(
 
         series_id, episode_number = video_id.split('.')
         episode_info = self._download_json(


@@ -143,29 +136,29 @@ class DramaFeverIE(DramaFeverBaseIE):
             video_id, 'Downloading episode info JSON', fatal=False)
         if episode_info:
             value = episode_info.get('value')
             video_id, 'Downloading episode info JSON', fatal=False)
         if episode_info:
             value = episode_info.get('value')

-            if value:
-                subfile = value[0].get('subfile') or value[0].get('new_subfile')
-                if subfile and subfile != 'http://www.dramafever.com/st/':
-                    subtitles.setdefault('English', []).append({
-                        'ext': 'srt',
-                        'url': subfile,
-                    })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'timestamp': timestamp,
-            'duration': duration,
-            'formats': formats,
-            'subtitles': subtitles,
-        }
+            if isinstance(value, list):
+                for v in value:
+                    if v.get('type') == 'Episode':
+                        subfile = v.get('subfile') or v.get('new_subfile')
+                        if subfile and subfile != 'http://www.dramafever.com/st/':
+                            info.setdefault('subtitles', {}).setdefault('English', []).append({
+                                'ext': 'srt',
+                                'url': subfile,
+                            })
+                        episode_number = int_or_none(v.get('number'))
+                        episode_fallback = 'Episode'
+                        if episode_number:
+                            episode_fallback += ' %d' % episode_number
+                        info['episode'] = v.get('title') or episode_fallback
+                        info['episode_number'] = episode_number
+                        break
+
+        return info

 
 
 class DramaFeverSeriesIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever:series'
 
 
 class DramaFeverSeriesIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever:series'

-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'

     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {