Update upstream source from tag 'upstream/2017.11.06'

[youtubedl] / youtube_dl / extractor / dramafever.py

diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index d35e88881d859555d20914240fff733508a4ac42..95883a037f537de47a0ee9f3f18d913eb9ec1647 100644 (file)
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8

 from __future__ import unicode_literals
 
 import itertools
 from __future__ import unicode_literals
 
 import itertools


@@ -6,20 +6,22 @@ import itertools
 from .amp import AMPIE
 from ..compat import (
     compat_HTTPError,
 from .amp import AMPIE
 from ..compat import (
     compat_HTTPError,

-    compat_urllib_parse,

     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
     clean_html,
     int_or_none,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
     clean_html,
     int_or_none,

+    remove_end,

     sanitized_Request,
     sanitized_Request,

+    urlencode_postdata

 )
 
 
 class DramaFeverBaseIE(AMPIE):
     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'
 )
 
 
 class DramaFeverBaseIE(AMPIE):
     _LOGIN_URL = 'https://www.dramafever.com/accounts/login/'
     _NETRC_MACHINE = 'dramafever'

+    _GEO_COUNTRIES = ['US', 'CA']

 
     _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
 
 
     _CONSUMER_SECRET = 'DA59dtVXYLxajktV'
 


@@ -50,14 +52,14 @@ class DramaFeverBaseIE(AMPIE):
         }
 
         request = sanitized_Request(
         }
 
         request = sanitized_Request(

-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_URL, urlencode_postdata(login_form))

         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
         if all(logout_pattern not in response
                for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
             error = self._html_search_regex(
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
         if all(logout_pattern not in response
                for logout_pattern in ['href="/accounts/logout/"', '>Log out<']):
             error = self._html_search_regex(

-                r'(?s)class="hidden-xs prompt"[^>]*>(.+?)<',
+                r'(?s)<h\d[^>]+\bclass="hidden-xs prompt"[^>]*>(.+?)</h\d',

                 response, 'error message', default=None)
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)
                 response, 'error message', default=None)
             if error:
                 raise ExtractorError('Unable to login: %s' % error, expected=True)


@@ -66,20 +68,20 @@ class DramaFeverBaseIE(AMPIE):
 
 class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'
 
 class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'

-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'

     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
             'id': '4512.1',

-            'ext': 'mp4',
-            'title': 'Cooking with Shin 4512.1',
+            'ext': 'flv',
+            'title': 'Cooking with Shin',

             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
             'episode': 'Episode 1',
             'episode_number': 1,
             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
             'episode': 'Episode 1',
             'episode_number': 1,

-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',

             'timestamp': 1404336058,
             'upload_date': '20140702',
             'timestamp': 1404336058,
             'upload_date': '20140702',

-            'duration': 343,
+            'duration': 344,

         },
         'params': {
             # m3u8 download
         },
         'params': {
             # m3u8 download


@@ -89,20 +91,23 @@ class DramaFeverIE(DramaFeverBaseIE):
         'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1',
         'info_dict': {
             'id': '4826.4',
         'url': 'http://www.dramafever.com/drama/4826/4/Mnet_Asian_Music_Awards_2015/?ap=1',
         'info_dict': {
             'id': '4826.4',

-            'ext': 'mp4',
-            'title': 'Mnet Asian Music Awards 2015 4826.4',
+            'ext': 'flv',
+            'title': 'Mnet Asian Music Awards 2015',

             'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
             'episode': 'Mnet Asian Music Awards 2015 - Part 3',
             'episode_number': 4,
             'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
             'episode': 'Mnet Asian Music Awards 2015 - Part 3',
             'episode_number': 4,

-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',

             'timestamp': 1450213200,
             'upload_date': '20151215',
             'timestamp': 1450213200,
             'upload_date': '20151215',

-            'duration': 5602,
+            'duration': 5359,

         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },

+    }, {
+        'url': 'https://www.dramafever.com/zh-cn/drama/4972/15/Doctor_Romantic/',
+        'only_matching': True,

     }]
 
     def _real_extract(self, url):
     }]
 
     def _real_extract(self, url):


@@ -113,10 +118,15 @@ class DramaFeverIE(DramaFeverBaseIE):
                 'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
                 'http://www.dramafever.com/amp/episode/feed.json?guid=%s' % video_id)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):

-                raise ExtractorError(
-                    'Currently unavailable in your country.', expected=True)
+                self.raise_geo_restricted(
+                    msg='Currently unavailable in your country',
+                    countries=self._GEO_COUNTRIES)

             raise
 
             raise
 

+        # title is postfixed with video id for some reason, removing
+        if info.get('title'):
+            info['title'] = remove_end(info['title'], video_id).strip()
+

         series_id, episode_number = video_id.split('.')
         episode_info = self._download_json(
             # We only need a single episode info, so restricting page size to one episode
         series_id, episode_number = video_id.split('.')
         episode_info = self._download_json(
             # We only need a single episode info, so restricting page size to one episode


@@ -148,7 +158,7 @@ class DramaFeverIE(DramaFeverBaseIE):
 
 class DramaFeverSeriesIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever:series'
 
 class DramaFeverSeriesIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever:series'

-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'

     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {