debian/README.source: Change 'markup'.

[youtubedl] / youtube_dl / extractor / videomore.py

diff --git a/youtube_dl/extractor/videomore.py b/youtube_dl/extractor/videomore.py
index 0bd1e1eec5212f9b2bb101870765b014c062d011..e3eda3327f48465a307390e23575d8754693737c 100644 (file)
--- a/youtube_dl/extractor/videomore.py
+++ b/youtube_dl/extractor/videomore.py
@@ -4,51 +4,62 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
 import re
 
 from .common import InfoExtractor

+from ..compat import compat_str

 from ..utils import (
     int_or_none,
 from ..utils import (
     int_or_none,

-    parse_age_limit,
-    parse_iso8601,
+    orderedSet,
+    parse_duration,
+    str_or_none,
+    unified_strdate,
+    url_or_none,
+    xpath_element,

     xpath_text,
 )
 
 
 class VideomoreIE(InfoExtractor):
     IE_NAME = 'videomore'
     xpath_text,
 )
 
 
 class VideomoreIE(InfoExtractor):
     IE_NAME = 'videomore'

-    _VALID_URL = r'videomore:(?P<sid>\d+)$|https?://videomore\.ru/(?:(?:embed|[^/]+/[^/]+)/|[^/]+\?.*\btrack_id=)(?P<id>\d+)(?:[/?#&]|\.(?:xml|json)|$)'
+    _VALID_URL = r'''(?x)
+                    videomore:(?P<sid>\d+)$|
+                    https?://(?:player\.)?videomore\.ru/
+                        (?:
+                            (?:
+                                embed|
+                                [^/]+/[^/]+
+                            )/|
+                            [^/]*\?.*?\btrack_id=
+                        )
+                        (?P<id>\d+)
+                        (?:[/?#&]|\.(?:xml|json)|$)
+                    '''

     _TESTS = [{
         'url': 'http://videomore.ru/kino_v_detalayah/5_sezon/367617',
     _TESTS = [{
         'url': 'http://videomore.ru/kino_v_detalayah/5_sezon/367617',

-        'md5': '70875fbf57a1cd004709920381587185',
+        'md5': '44455a346edc0d509ac5b5a5b531dc35',

         'info_dict': {
             'id': '367617',
             'ext': 'flv',
         'info_dict': {
             'id': '367617',
             'ext': 'flv',

-            'title': 'В гостях Алексей Чумаков и Юлия Ковальчук',
-            'description': 'В гостях – лучшие романтические комедии года, «Выживший» Иньярриту и «Стив Джобс» Дэнни Бойла.',
+            'title': 'Кино в деталях 5 сезон В гостях Алексей Чумаков и Юлия Ковальчук',

             'series': 'Кино в деталях',
             'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук',
             'series': 'Кино в деталях',
             'episode': 'В гостях Алексей Чумаков и Юлия Ковальчук',

-            'episode_number': None,
-            'season': 'Сезон 2015',
-            'season_number': 5,
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',

             'duration': 2910,
             'duration': 2910,

-            'age_limit': 16,

             'view_count': int,
             'view_count': int,

+            'comment_count': int,
+            'age_limit': 16,

         },
     }, {
         'url': 'http://videomore.ru/embed/259974',
         'info_dict': {
             'id': '259974',
             'ext': 'flv',
         },
     }, {
         'url': 'http://videomore.ru/embed/259974',
         'info_dict': {
             'id': '259974',
             'ext': 'flv',

-            'title': '80 серия',
-            'description': '«Медведей» ждет решающий матч. Макеев выясняет отношения со Стрельцовым. Парни узнают подробности прошлого Макеева.',
+            'title': 'Молодежка 2 сезон 40 серия',

             'series': 'Молодежка',
             'series': 'Молодежка',

-            'episode': '80 серия',
-            'episode_number': 40,
-            'season': '2 сезон',
-            'season_number': 2,
-            'thumbnail': 're:^https?://.*\.jpg',
+            'episode': '40 серия',
+            'thumbnail': r're:^https?://.*\.jpg',

             'duration': 2809,
             'duration': 2809,

-            'age_limit': 16,

             'view_count': int,
             'view_count': int,

+            'comment_count': int,
+            'age_limit': 16,

         },
         'params': {
             'skip_download': True,
         },
         'params': {
             'skip_download': True,


@@ -58,14 +69,9 @@ class VideomoreIE(InfoExtractor):
         'info_dict': {
             'id': '341073',
             'ext': 'flv',
         'info_dict': {
             'id': '341073',
             'ext': 'flv',

-            'title': 'Команда проиграла из-за Бакина?',
-            'description': 'Молодежка 3 сезон скоро',
-            'series': 'Молодежка',
+            'title': 'Промо Команда проиграла из-за Бакина?',

             'episode': 'Команда проиграла из-за Бакина?',
             'episode': 'Команда проиграла из-за Бакина?',

-            'episode_number': None,
-            'season': 'Промо',
-            'season_number': 99,
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',

             'duration': 29,
             'age_limit': 16,
             'view_count': int,
             'duration': 29,
             'age_limit': 16,
             'view_count': int,


@@ -91,13 +97,21 @@ class VideomoreIE(InfoExtractor):
     }, {
         'url': 'videomore:367617',
         'only_matching': True,
     }, {
         'url': 'videomore:367617',
         'only_matching': True,

+    }, {
+        'url': 'https://player.videomore.ru/?partner_id=97&track_id=736234&autoplay=0&userToken=',
+        'only_matching': True,

     }]
 
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
     }]
 
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(

-            r'<object[^>]+data=(["\'])https?://videomore.ru/player\.swf\?.*config=(?P<url>https?://videomore\.ru/(?:[^/]+/)+\d+\.xml).*\1',
+            r'<object[^>]+data=(["\'])https?://videomore\.ru/player\.swf\?.*config=(?P<url>https?://videomore\.ru/(?:[^/]+/)+\d+\.xml).*\1',

             webpage)
             webpage)

+        if not mobj:
+            mobj = re.search(
+                r'<iframe[^>]+src=([\'"])(?P<url>https?://videomore\.ru/embed/\d+)',
+                webpage)
+

         if mobj:
             return mobj.group('url')
 
         if mobj:
             return mobj.group('url')
 


@@ -109,42 +123,33 @@ class VideomoreIE(InfoExtractor):
             'http://videomore.ru/video/tracks/%s.xml' % video_id,
             video_id, 'Downloading video XML')
 
             'http://videomore.ru/video/tracks/%s.xml' % video_id,
             video_id, 'Downloading video XML')
 

-        video_url = xpath_text(video, './/video_url', 'video url', fatal=True)
+        item = xpath_element(video, './/playlist/item', fatal=True)
+
+        title = xpath_text(
+            item, ('./title', './episode_name'), 'title', fatal=True)
+
+        video_url = xpath_text(item, './video_url', 'video url', fatal=True)

         formats = self._extract_f4m_formats(video_url, video_id, f4m_id='hds')
         formats = self._extract_f4m_formats(video_url, video_id, f4m_id='hds')

+        self._sort_formats(formats)
+
+        thumbnail = xpath_text(item, './thumbnail_url')
+        duration = int_or_none(xpath_text(item, './duration'))
+        view_count = int_or_none(xpath_text(item, './views'))
+        comment_count = int_or_none(xpath_text(item, './count_comments'))
+        age_limit = int_or_none(xpath_text(item, './min_age'))

 
 

-        data = self._download_json(
-            'http://videomore.ru/video/tracks/%s.json' % video_id,
-            video_id, 'Downloading video JSON')
-
-        title = data.get('title') or data['project_title']
-        description = data.get('description') or data.get('description_raw')
-        timestamp = parse_iso8601(data.get('published_at'))
-        duration = int_or_none(data.get('duration'))
-        view_count = int_or_none(data.get('views'))
-        age_limit = parse_age_limit(data.get('min_age'))
-        thumbnails = [{
-            'url': thumbnail,
-        } for thumbnail in data.get('big_thumbnail_urls', [])]
-
-        series = data.get('project_title')
-        episode = data.get('title')
-        episode_number = int_or_none(data.get('episode_of_season') or None)
-        season = data.get('season_title')
-        season_number = int_or_none(data.get('season_pos') or None)
+        series = xpath_text(item, './project_name')
+        episode = xpath_text(item, './episode_name')

 
         return {
             'id': video_id,
             'title': title,
 
         return {
             'id': video_id,
             'title': title,

-            'description': description,

             'series': series,
             'episode': episode,
             'series': series,
             'episode': episode,

-            'episode_number': episode_number,
-            'season': season,
-            'season_number': season_number,
-            'thumbnails': thumbnails,
-            'timestamp': timestamp,
+            'thumbnail': thumbnail,

             'duration': duration,
             'view_count': view_count,
             'duration': duration,
             'view_count': view_count,

+            'comment_count': comment_count,

             'age_limit': age_limit,
             'formats': formats,
         }
             'age_limit': age_limit,
             'formats': formats,
         }


@@ -152,7 +157,7 @@ class VideomoreIE(InfoExtractor):
 
 class VideomoreVideoIE(InfoExtractor):
     IE_NAME = 'videomore:video'
 
 class VideomoreVideoIE(InfoExtractor):
     IE_NAME = 'videomore:video'

-    _VALID_URL = r'https?://videomore\.ru/(?:(?:[^/]+/){2})?(?P<id>[^/?#&]+)[/?#&]*$'
+    _VALID_URL = r'https?://videomore\.ru/(?:(?:[^/]+/){2})?(?P<id>[^/?#&]+)(?:/*|[?#&].*?)$'

     _TESTS = [{
         # single video with og:video:iframe
         'url': 'http://videomore.ru/elki_3',
     _TESTS = [{
         # single video with og:video:iframe
         'url': 'http://videomore.ru/elki_3',


@@ -161,7 +166,7 @@ class VideomoreVideoIE(InfoExtractor):
             'ext': 'flv',
             'title': 'Ёлки 3',
             'description': '',
             'ext': 'flv',
             'title': 'Ёлки 3',
             'description': '',

-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',

             'duration': 5579,
             'age_limit': 6,
             'view_count': int,
             'duration': 5579,
             'age_limit': 6,
             'view_count': int,


@@ -184,7 +189,7 @@ class VideomoreVideoIE(InfoExtractor):
             'ext': 'flv',
             'title': '1 серия. Здравствуй, Аквавилль!',
             'description': 'md5:c6003179538b5d353e7bcd5b1372b2d7',
             'ext': 'flv',
             'title': '1 серия. Здравствуй, Аквавилль!',
             'description': 'md5:c6003179538b5d353e7bcd5b1372b2d7',

-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',

             'duration': 754,
             'age_limit': 6,
             'view_count': int,
             'duration': 754,
             'age_limit': 6,
             'view_count': int,


@@ -192,6 +197,9 @@ class VideomoreVideoIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
         'params': {
             'skip_download': True,
         },

+    }, {
+        'url': 'https://videomore.ru/molodezhka/6_sezon/29_seriya?utm_so',
+        'only_matching': True,

     }]
 
     @classmethod
     }]
 
     @classmethod


@@ -212,13 +220,16 @@ class VideomoreVideoIE(InfoExtractor):
                  r'track-id=["\'](\d+)',
                  r'xcnt_product_id\s*=\s*(\d+)'), webpage, 'video id')
             video_url = 'videomore:%s' % video_id
                  r'track-id=["\'](\d+)',
                  r'xcnt_product_id\s*=\s*(\d+)'), webpage, 'video id')
             video_url = 'videomore:%s' % video_id

+        else:
+            video_id = None

 
 

-        return self.url_result(video_url, VideomoreIE.ie_key())
+        return self.url_result(
+            video_url, ie=VideomoreIE.ie_key(), video_id=video_id)

 
 
 class VideomoreSeasonIE(InfoExtractor):
     IE_NAME = 'videomore:season'
 
 
 class VideomoreSeasonIE(InfoExtractor):
     IE_NAME = 'videomore:season'

-    _VALID_URL = r'https?://videomore\.ru/(?!embed)(?P<id>[^/]+/[^/?#&]+)[/?#&]*$'
+    _VALID_URL = r'https?://videomore\.ru/(?!embed)(?P<id>[^/]+/[^/?#&]+)(?:/*|[?#&].*?)$'

     _TESTS = [{
         'url': 'http://videomore.ru/molodezhka/sezon_promo',
         'info_dict': {
     _TESTS = [{
         'url': 'http://videomore.ru/molodezhka/sezon_promo',
         'info_dict': {


@@ -226,8 +237,16 @@ class VideomoreSeasonIE(InfoExtractor):
             'title': 'Молодежка Промо',
         },
         'playlist_mincount': 12,
             'title': 'Молодежка Промо',
         },
         'playlist_mincount': 12,

+    }, {
+        'url': 'http://videomore.ru/molodezhka/sezon_promo?utm_so',
+        'only_matching': True,

     }]
 
     }]
 

+    @classmethod
+    def suitable(cls, url):
+        return (False if (VideomoreIE.suitable(url) or VideomoreVideoIE.suitable(url))
+                else super(VideomoreSeasonIE, cls).suitable(url))
+

     def _real_extract(self, url):
         display_id = self._match_id(url)
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
 


@@ -235,9 +254,54 @@ class VideomoreSeasonIE(InfoExtractor):
 
         title = self._og_search_title(webpage)
 
 
         title = self._og_search_title(webpage)
 

-        entries = [
-            self.url_result(item) for item in re.findall(
-                r'<a[^>]+href="((?:https?:)?//videomore\.ru/%s/[^/]+)"[^>]+class="widget-item-desc"'
-                % display_id, webpage)]
+        data = self._parse_json(
+            self._html_search_regex(
+                r'\bclass=["\']seasons-tracks["\'][^>]+\bdata-custom-data=(["\'])(?P<value>{.+?})\1',
+                webpage, 'data', default='{}', group='value'),
+            display_id, fatal=False)
+
+        entries = []
+
+        if data:
+            episodes = data.get('episodes')
+            if isinstance(episodes, list):
+                for ep in episodes:
+                    if not isinstance(ep, dict):
+                        continue
+                    ep_id = int_or_none(ep.get('id'))
+                    ep_url = url_or_none(ep.get('url'))
+                    if ep_id:
+                        e = {
+                            'url': 'videomore:%s' % ep_id,
+                            'id': compat_str(ep_id),
+                        }
+                    elif ep_url:
+                        e = {'url': ep_url}
+                    else:
+                        continue
+                    e.update({
+                        '_type': 'url',
+                        'ie_key': VideomoreIE.ie_key(),
+                        'title': str_or_none(ep.get('title')),
+                        'thumbnail': url_or_none(ep.get('image')),
+                        'duration': parse_duration(ep.get('duration')),
+                        'episode_number': int_or_none(ep.get('number')),
+                        'upload_date': unified_strdate(ep.get('date')),
+                    })
+                    entries.append(e)
+
+        if not entries:
+            entries = [
+                self.url_result(
+                    'videomore:%s' % video_id, ie=VideomoreIE.ie_key(),
+                    video_id=video_id)
+                for video_id in orderedSet(re.findall(
+                    r':(?:id|key)=["\'](\d+)["\']', webpage))]
+
+        if not entries:
+            entries = [
+                self.url_result(item) for item in re.findall(
+                    r'<a[^>]+href="((?:https?:)?//videomore\.ru/%s/[^/]+)"[^>]+class="widget-item-desc"'
+                    % display_id, webpage)]

 
         return self.playlist_result(entries, display_id, title)
 
         return self.playlist_result(entries, display_id, title)