Update upstream source from tag 'upstream/2019.07.02'

[youtubedl] / youtube_dl / extractor / camdemy.py

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 897f3a104ce2d31aeac99e98197557ef502faf18..8f0c6c545c35312813bb461b3218f868a660fdd2 100644 (file)
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -1,22 +1,23 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
 # coding: utf-8
 from __future__ import unicode_literals
 

-import datetime

 import re
 
 from .common import InfoExtractor
 from ..compat import (
 import re
 
 from .common import InfoExtractor
 from ..compat import (

-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,

     compat_urlparse,
 )
 from ..utils import (
     compat_urlparse,
 )
 from ..utils import (

-    parse_iso8601,
+    clean_html,
+    parse_duration,

     str_to_int,
     str_to_int,

+    unified_strdate,

 )
 
 
 class CamdemyIE(InfoExtractor):
 )
 
 
 class CamdemyIE(InfoExtractor):

-    _VALID_URL = r'http://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'

     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',
     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',


@@ -25,86 +26,93 @@ class CamdemyIE(InfoExtractor):
             'id': '5181',
             'ext': 'mp4',
             'title': 'Ch1-1 Introduction, Signals (02-23-2012)',
             'id': '5181',
             'ext': 'mp4',
             'title': 'Ch1-1 Introduction, Signals (02-23-2012)',

-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': '',
+            'thumbnail': r're:^https?://.*\.jpg$',

             'creator': 'ss11spring',
             'creator': 'ss11spring',

+            'duration': 1591,

             'upload_date': '20130114',
             'upload_date': '20130114',

-            'timestamp': 1358154556,

             'view_count': int,
         }
     }, {
         # With non-empty description
             'view_count': int,
         }
     }, {
         # With non-empty description

+        # webpage returns "No permission or not login"

         'url': 'http://www.camdemy.com/media/13885',
         'md5': '4576a3bb2581f86c61044822adbd1249',
         'info_dict': {
             'id': '13885',
             'ext': 'mp4',
             'title': 'EverCam + Camdemy QuickStart',
         'url': 'http://www.camdemy.com/media/13885',
         'md5': '4576a3bb2581f86c61044822adbd1249',
         'info_dict': {
             'id': '13885',
             'ext': 'mp4',
             'title': 'EverCam + Camdemy QuickStart',

-            'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 'md5:050b62f71ed62928f8a35f1a41e186c9',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:2a9f989c2b153a2342acee579c6e7db6',

             'creator': 'evercam',
             'creator': 'evercam',

-            'upload_date': '20140620',
-            'timestamp': 1403271569,
+            'duration': 318,

         }
     }, {
         }
     }, {

-        # External source
+        # External source (YouTube)

         'url': 'http://www.camdemy.com/media/14842',
         'url': 'http://www.camdemy.com/media/14842',

-        'md5': '50e1c3c3aa233d3d7b7daa2fa10b1cf7',

         'info_dict': {
             'id': '2vsYQzNIsJo',
             'ext': 'mp4',
         'info_dict': {
             'id': '2vsYQzNIsJo',
             'ext': 'mp4',

+            'title': 'Excel 2013 Tutorial - How to add Password Protection',
+            'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection',

             'upload_date': '20130211',
             'uploader': 'Hun Kim',
             'upload_date': '20130211',
             'uploader': 'Hun Kim',

-            'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection',

             'uploader_id': 'hunkimtutorials',
             'uploader_id': 'hunkimtutorials',

-            'title': 'Excel 2013 Tutorial - How to add Password Protection',
-        }
+        },
+        'params': {
+            'skip_download': True,
+        },

     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

-        page = self._download_webpage(url, video_id)
+
+        webpage = self._download_webpage(url, video_id)

 
         src_from = self._html_search_regex(
 
         src_from = self._html_search_regex(

-            r"<div class='srcFrom'>Source: <a title='([^']+)'", page,
-            'external source', default=None)
+            r"class=['\"]srcFrom['\"][^>]*>Sources?(?:\s+from)?\s*:\s*<a[^>]+(?:href|title)=(['\"])(?P<url>(?:(?!\1).)+)\1",
+            webpage, 'external source', default=None, group='url')

         if src_from:
             return self.url_result(src_from)
 
         oembed_obj = self._download_json(
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         if src_from:
             return self.url_result(src_from)
 
         oembed_obj = self._download_json(
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 

+        title = oembed_obj['title']

         thumb_url = oembed_obj['thumbnail_url']
         video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
         file_list_doc = self._download_xml(
             compat_urlparse.urljoin(video_folder, 'fileList.xml'),
         thumb_url = oembed_obj['thumbnail_url']
         video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
         file_list_doc = self._download_xml(
             compat_urlparse.urljoin(video_folder, 'fileList.xml'),

-            video_id, 'Filelist XML')
+            video_id, 'Downloading filelist XML')

         file_name = file_list_doc.find('./video/item/fileName').text
         video_url = compat_urlparse.urljoin(video_folder, file_name)
 
         file_name = file_list_doc.find('./video/item/fileName').text
         video_url = compat_urlparse.urljoin(video_folder, file_name)
 

-        timestamp = parse_iso8601(self._html_search_regex(
-            r"<div class='title'>Posted\s*:</div>\s*<div class='value'>([^<>]+)<",
-            page, 'creation time', fatal=False),
-            delimiter=' ', timezone=datetime.timedelta(hours=8))
-        view_count = str_to_int(self._html_search_regex(
-            r"<div class='title'>Views\s*:</div>\s*<div class='value'>([^<>]+)<",
-            page, 'view count', fatal=False))
+        # Some URLs return "No permission or not login" in a webpage despite being
+        # freely available via oembed JSON URL (e.g. http://www.camdemy.com/media/13885)
+        upload_date = unified_strdate(self._search_regex(
+            r'>published on ([^<]+)<', webpage,
+            'upload date', default=None))
+        view_count = str_to_int(self._search_regex(
+            r'role=["\']viewCnt["\'][^>]*>([\d,.]+) views',
+            webpage, 'view count', default=None))
+        description = self._html_search_meta(
+            'description', webpage, default=None) or clean_html(
+            oembed_obj.get('description'))

 
         return {
             'id': video_id,
             'url': video_url,
 
         return {
             'id': video_id,
             'url': video_url,

-            'title': oembed_obj['title'],
+            'title': title,

             'thumbnail': thumb_url,
             'thumbnail': thumb_url,

-            'description': self._html_search_meta('description', page),
-            'creator': oembed_obj['author_name'],
-            'duration': oembed_obj['duration'],
-            'timestamp': timestamp,
+            'description': description,
+            'creator': oembed_obj.get('author_name'),
+            'duration': parse_duration(oembed_obj.get('duration')),
+            'upload_date': upload_date,

             'view_count': view_count,
         }
 
 
 class CamdemyFolderIE(InfoExtractor):
             'view_count': view_count,
         }
 
 
 class CamdemyFolderIE(InfoExtractor):

-    _VALID_URL = r'http://www.camdemy.com/folder/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?camdemy\.com/folder/(?P<id>\d+)'

     _TESTS = [{
         # links with trailing slash
         'url': 'http://www.camdemy.com/folder/450',
     _TESTS = [{
         # links with trailing slash
         'url': 'http://www.camdemy.com/folder/450',


@@ -139,7 +147,7 @@ class CamdemyFolderIE(InfoExtractor):
         parsed_url = list(compat_urlparse.urlparse(url))
         query = dict(compat_urlparse.parse_qsl(parsed_url[4]))
         query.update({'displayMode': 'list'})
         parsed_url = list(compat_urlparse.urlparse(url))
         query = dict(compat_urlparse.parse_qsl(parsed_url[4]))
         query.update({'displayMode': 'list'})

-        parsed_url[4] = compat_urllib_parse.urlencode(query)
+        parsed_url[4] = compat_urllib_parse_urlencode(query)

         final_url = compat_urlparse.urlunparse(parsed_url)
 
         page = self._download_webpage(final_url, folder_id)
         final_url = compat_urlparse.urlunparse(parsed_url)
 
         page = self._download_webpage(final_url, folder_id)