New upstream version 2017.05.18.1

[youtubedl] / youtube_dl / extractor / niconico.py

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 0f8aa5adad5b2247621ce00249f3bd03a33a104a..8baac23e4b16643a59e6a83570862b6a5afd2b45 100644 (file)
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -1,4 +1,4 @@
-# encoding: utf-8
+# coding: utf-8

 from __future__ import unicode_literals
 
 import re
 from __future__ import unicode_literals
 
 import re


@@ -7,8 +7,6 @@ import datetime
 
 from .common import InfoExtractor
 from ..compat import (
 
 from .common import InfoExtractor
 from ..compat import (

-    compat_urllib_parse,
-    compat_urllib_request,

     compat_urlparse,
 )
 from ..utils import (
     compat_urlparse,
 )
 from ..utils import (


@@ -16,8 +14,10 @@ from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
     int_or_none,
     parse_duration,
     parse_iso8601,

+    sanitized_Request,

     xpath_text,
     determine_ext,
     xpath_text,
     determine_ext,

+    urlencode_postdata,

 )
 
 
 )
 
 


@@ -39,6 +39,7 @@ class NiconicoIE(InfoExtractor):
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
         },
             'description': '(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org',
             'duration': 33,
         },

+        'skip': 'Requires an account',

     }, {
         # File downloaded with and without credentials are different, so omit
         # the md5 field
     }, {
         # File downloaded with and without credentials are different, so omit
         # the md5 field


@@ -54,6 +55,7 @@ class NiconicoIE(InfoExtractor):
             'timestamp': 1304065916,
             'duration': 209,
         },
             'timestamp': 1304065916,
             'duration': 209,
         },

+        'skip': 'Requires an account',

     }, {
         # 'video exists but is marked as "deleted"
         # md5 is unstable
     }, {
         # 'video exists but is marked as "deleted"
         # md5 is unstable


@@ -64,9 +66,10 @@ class NiconicoIE(InfoExtractor):
             'description': 'deleted',
             'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
             'upload_date': '20071224',
             'description': 'deleted',
             'title': 'ドラえもんエターナル第3話「決戦第3新東京市」＜前編＞',
             'upload_date': '20071224',

-            'timestamp': 1198527840,  # timestamp field has different value if logged in
+            'timestamp': int,  # timestamp field has different value if logged in

             'duration': 304,
         },
             'duration': 304,
         },

+        'skip': 'Requires an account',

     }, {
         'url': 'http://www.nicovideo.jp/watch/so22543406',
         'info_dict': {
     }, {
         'url': 'http://www.nicovideo.jp/watch/so22543406',
         'info_dict': {


@@ -78,13 +81,12 @@ class NiconicoIE(InfoExtractor):
             'upload_date': '20140104',
             'uploader': 'アニメロチャンネル',
             'uploader_id': '312',
             'upload_date': '20140104',
             'uploader': 'アニメロチャンネル',
             'uploader_id': '312',

-        }
+        },
+        'skip': 'The viewing period of the video you were searching for has expired.',

     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'
     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'

-    # Determine whether the downloader used authentication to download video
-    _AUTHENTICATED = False

 
     def _real_initialize(self):
         self._login()
 
     def _real_initialize(self):
         self._login()


@@ -100,19 +102,14 @@ class NiconicoIE(InfoExtractor):
             'mail': username,
             'password': password,
         }
             'mail': username,
             'password': password,
         }

-        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
-        # chokes on unicode
-        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
-        login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
-        request = compat_urllib_request.Request(
+        login_data = urlencode_postdata(login_form_strs)
+        request = sanitized_Request(

             'https://secure.nicovideo.jp/secure/login', login_data)
         login_results = self._download_webpage(
             request, None, note='Logging in', errnote='Unable to log in')
         if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None:
             self._downloader.report_warning('unable to log in: bad username or password')
             return False
             'https://secure.nicovideo.jp/secure/login', login_data)
         login_results = self._download_webpage(
             request, None, note='Logging in', errnote='Unable to log in')
         if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None:
             self._downloader.report_warning('unable to log in: bad username or password')
             return False

-        # Successful login
-        self._AUTHENTICATED = True

         return True
 
     def _real_extract(self, url):
         return True
 
     def _real_extract(self, url):


@@ -130,35 +127,19 @@ class NiconicoIE(InfoExtractor):
             'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
             note='Downloading video info page')
 
             'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
             note='Downloading video info page')
 

-        if self._AUTHENTICATED:
-            # Get flv info
-            flv_info_webpage = self._download_webpage(
-                'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
-                video_id, 'Downloading flv info')
-        else:
-            # Get external player info
-            ext_player_info = self._download_webpage(
-                'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
-            thumb_play_key = self._search_regex(
-                r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
-
-            # Get flv info
-            flv_info_data = compat_urllib_parse.urlencode({
-                'k': thumb_play_key,
-                'v': video_id
-            })
-            flv_info_request = compat_urllib_request.Request(
-                'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
-                {'Content-Type': 'application/x-www-form-urlencoded'})
-            flv_info_webpage = self._download_webpage(
-                flv_info_request, video_id,
-                note='Downloading flv info', errnote='Unable to download flv info')
+        # Get flv info
+        flv_info_webpage = self._download_webpage(
+            'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
+            video_id, 'Downloading flv info')

 
         flv_info = compat_urlparse.parse_qs(flv_info_webpage)
         if 'url' not in flv_info:
             if 'deleted' in flv_info:
                 raise ExtractorError('The video has been deleted.',
                                      expected=True)
 
         flv_info = compat_urlparse.parse_qs(flv_info_webpage)
         if 'url' not in flv_info:
             if 'deleted' in flv_info:
                 raise ExtractorError('The video has been deleted.',
                                      expected=True)

+            elif 'closed' in flv_info:
+                raise ExtractorError('Niconico videos now require logging in',
+                                     expected=True)

             else:
                 raise ExtractorError('Unable to find video URL')
 
             else:
                 raise ExtractorError('Unable to find video URL')
 


@@ -254,7 +235,7 @@ class NiconicoIE(InfoExtractor):
 
 
 class NiconicoPlaylistIE(InfoExtractor):
 
 
 class NiconicoPlaylistIE(InfoExtractor):

-    _VALID_URL = r'https?://www\.nicovideo\.jp/mylist/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/mylist/(?P<id>\d+)'

 
     _TEST = {
         'url': 'http://www.nicovideo.jp/mylist/27411728',
 
     _TEST = {
         'url': 'http://www.nicovideo.jp/mylist/27411728',