debian/patches: Add patch to skip style checks from flake8 during test.

[youtubedl] / youtube_dl / extractor / onet.py

diff --git a/youtube_dl/extractor/onet.py b/youtube_dl/extractor/onet.py
index 0a501b3e5b9f0c0bd2c6789e4da302b7610ced28..e55b2ac89bb7b6449d0ac51f1092d30742b3606c 100644 (file)
--- a/youtube_dl/extractor/onet.py
+++ b/youtube_dl/extractor/onet.py
@@ -11,6 +11,7 @@ from ..utils import (
     get_element_by_class,
     int_or_none,
     js_to_json,
     get_element_by_class,
     int_or_none,
     js_to_json,

+    NO_DEFAULT,

     parse_iso8601,
     remove_start,
     strip_or_none,
     parse_iso8601,
     remove_start,
     strip_or_none,


@@ -19,11 +20,13 @@ from ..utils import (
 
 
 class OnetBaseIE(InfoExtractor):
 
 
 class OnetBaseIE(InfoExtractor):

+    _URL_BASE_RE = r'https?://(?:(?:www\.)?onet\.tv|onet100\.vod\.pl)/[a-z]/'
+

     def _search_mvp_id(self, webpage):
         return self._search_regex(
             r'id=(["\'])mvp:(?P<id>.+?)\1', webpage, 'mvp id', group='id')
 
     def _search_mvp_id(self, webpage):
         return self._search_regex(
             r'id=(["\'])mvp:(?P<id>.+?)\1', webpage, 'mvp id', group='id')
 

-    def _extract_from_id(self, video_id, webpage):
+    def _extract_from_id(self, video_id, webpage=None):

         response = self._download_json(
             'http://qi.ckm.onetapi.pl/', video_id,
             query={
         response = self._download_json(
             'http://qi.ckm.onetapi.pl/', video_id,
             query={


@@ -44,7 +47,7 @@ class OnetBaseIE(InfoExtractor):
         video = response['result'].get('0')
 
         formats = []
         video = response['result'].get('0')
 
         formats = []

-        for _, formats_dict in video['formats'].items():
+        for format_type, formats_dict in video['formats'].items():

             if not isinstance(formats_dict, dict):
                 continue
             for format_id, format_list in formats_dict.items():
             if not isinstance(formats_dict, dict):
                 continue
             for format_id, format_list in formats_dict.items():


@@ -55,27 +58,39 @@ class OnetBaseIE(InfoExtractor):
                     if not video_url:
                         continue
                     ext = determine_ext(video_url)
                     if not video_url:
                         continue
                     ext = determine_ext(video_url)

-                    if format_id == 'ism':
+                    if format_id.startswith('ism'):

                         formats.extend(self._extract_ism_formats(
                             video_url, video_id, 'mss', fatal=False))
                     elif ext == 'mpd':
                         formats.extend(self._extract_mpd_formats(
                             video_url, video_id, mpd_id='dash', fatal=False))
                         formats.extend(self._extract_ism_formats(
                             video_url, video_id, 'mss', fatal=False))
                     elif ext == 'mpd':
                         formats.extend(self._extract_mpd_formats(
                             video_url, video_id, mpd_id='dash', fatal=False))

+                    elif format_id.startswith('hls'):
+                        formats.extend(self._extract_m3u8_formats(
+                            video_url, video_id, 'mp4', 'm3u8_native',
+                            m3u8_id='hls', fatal=False))

                     else:
                     else:

-                        formats.append({
+                        http_f = {

                             'url': video_url,
                             'format_id': format_id,
                             'url': video_url,
                             'format_id': format_id,

-                            'height': int_or_none(f.get('vertical_resolution')),
-                            'width': int_or_none(f.get('horizontal_resolution')),

                             'abr': float_or_none(f.get('audio_bitrate')),
                             'abr': float_or_none(f.get('audio_bitrate')),

-                            'vbr': float_or_none(f.get('video_bitrate')),
-                        })
+                        }
+                        if format_type == 'audio':
+                            http_f['vcodec'] = 'none'
+                        else:
+                            http_f.update({
+                                'height': int_or_none(f.get('vertical_resolution')),
+                                'width': int_or_none(f.get('horizontal_resolution')),
+                                'vbr': float_or_none(f.get('video_bitrate')),
+                            })
+                        formats.append(http_f)

         self._sort_formats(formats)
 
         meta = video.get('meta', {})
 
         self._sort_formats(formats)
 
         meta = video.get('meta', {})
 

-        title = self._og_search_title(webpage, default=None) or meta['title']
-        description = self._og_search_description(webpage, default=None) or meta.get('description')
+        title = (self._og_search_title(
+            webpage, default=None) if webpage else None) or meta['title']
+        description = (self._og_search_description(
+            webpage, default=None) if webpage else None) or meta.get('description')

         duration = meta.get('length') or meta.get('lenght')
         timestamp = parse_iso8601(meta.get('addDate'), ' ')
 
         duration = meta.get('length') or meta.get('lenght')
         timestamp = parse_iso8601(meta.get('addDate'), ' ')
 


@@ -89,13 +104,25 @@ class OnetBaseIE(InfoExtractor):
         }
 
 
         }
 
 

+class OnetMVPIE(OnetBaseIE):
+    _VALID_URL = r'onetmvp:(?P<id>\d+\.\d+)'
+
+    _TEST = {
+        'url': 'onetmvp:381027.1509591944',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        return self._extract_from_id(self._match_id(url))
+
+

 class OnetIE(OnetBaseIE):
 class OnetIE(OnetBaseIE):

-    _VALID_URL = r'https?://(?:www\.)?onet\.tv/[a-z]/[a-z]+/(?P<display_id>[0-9a-z-]+)/(?P<id>[0-9a-z]+)'
+    _VALID_URL = OnetBaseIE._URL_BASE_RE + r'[a-z]+/(?P<display_id>[0-9a-z-]+)/(?P<id>[0-9a-z]+)'

     IE_NAME = 'onet.tv'
 
     IE_NAME = 'onet.tv'
 

-    _TEST = {
+    _TESTS = [{

         'url': 'http://onet.tv/k/openerfestival/open-er-festival-2016-najdziwniejsze-wymagania-gwiazd/qbpyqc',
         'url': 'http://onet.tv/k/openerfestival/open-er-festival-2016-najdziwniejsze-wymagania-gwiazd/qbpyqc',

-        'md5': 'e3ffbf47590032ac3f27249204173d50',
+        'md5': '436102770fb095c75b8bb0392d3da9ff',

         'info_dict': {
             'id': 'qbpyqc',
             'display_id': 'open-er-festival-2016-najdziwniejsze-wymagania-gwiazd',
         'info_dict': {
             'id': 'qbpyqc',
             'display_id': 'open-er-festival-2016-najdziwniejsze-wymagania-gwiazd',


@@ -105,7 +132,10 @@ class OnetIE(OnetBaseIE):
             'upload_date': '20160705',
             'timestamp': 1467721580,
         },
             'upload_date': '20160705',
             'timestamp': 1467721580,
         },

-    }
+    }, {
+        'url': 'https://onet100.vod.pl/k/openerfestival/open-er-festival-2016-najdziwniejsze-wymagania-gwiazd/qbpyqc',
+        'only_matching': True,
+    }]

 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)


@@ -125,18 +155,21 @@ class OnetIE(OnetBaseIE):
 
 
 class OnetChannelIE(OnetBaseIE):
 
 
 class OnetChannelIE(OnetBaseIE):

-    _VALID_URL = r'https?://(?:www\.)?onet\.tv/[a-z]/(?P<id>[a-z]+)(?:[?#]|$)'
+    _VALID_URL = OnetBaseIE._URL_BASE_RE + r'(?P<id>[a-z]+)(?:[?#]|$)'

     IE_NAME = 'onet.tv:channel'
 
     IE_NAME = 'onet.tv:channel'
 

-    _TEST = {
+    _TESTS = [{

         'url': 'http://onet.tv/k/openerfestival',
         'info_dict': {
             'id': 'openerfestival',
         'url': 'http://onet.tv/k/openerfestival',
         'info_dict': {
             'id': 'openerfestival',

-            'title': 'Open\'er Festival Live',
-            'description': 'Dziękujemy, że oglądaliście transmisje. Zobaczcie nasze relacje i wywiady z artystami.',
+            'title': "Open'er Festival",
+            'description': "Tak było na Open'er Festival 2016! Oglądaj nasze reportaże i wywiady z artystami.",

         },
         },

-        'playlist_mincount': 46,
-    }
+        'playlist_mincount': 35,
+    }, {
+        'url': 'https://onet100.vod.pl/k/openerfestival',
+        'only_matching': True,
+    }]

 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)


@@ -158,7 +191,7 @@ class OnetChannelIE(OnetBaseIE):
             'Downloading channel %s - add --no-playlist to just download video %s' % (
                 channel_id, video_name))
         matches = re.findall(
             'Downloading channel %s - add --no-playlist to just download video %s' % (
                 channel_id, video_name))
         matches = re.findall(

-            r'<a[^>]+href=[\'"](https?://(?:www\.)?onet\.tv/[a-z]/[a-z]+/[0-9a-z-]+/[0-9a-z]+)',
+            r'<a[^>]+href=[\'"](%s[a-z]+/[0-9a-z-]+/[0-9a-z]+)' % self._URL_BASE_RE,

             webpage)
         entries = [
             self.url_result(video_link, OnetIE.ie_key())
             webpage)
         entries = [
             self.url_result(video_link, OnetIE.ie_key())


@@ -167,3 +200,69 @@ class OnetChannelIE(OnetBaseIE):
         channel_title = strip_or_none(get_element_by_class('o_channelName', webpage))
         channel_description = strip_or_none(get_element_by_class('o_channelDesc', webpage))
         return self.playlist_result(entries, channel_id, channel_title, channel_description)
         channel_title = strip_or_none(get_element_by_class('o_channelName', webpage))
         channel_description = strip_or_none(get_element_by_class('o_channelDesc', webpage))
         return self.playlist_result(entries, channel_id, channel_title, channel_description)

+
+
+class OnetPlIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[^/]+\.)?(?:onet|businessinsider\.com|plejada)\.pl/(?:[^/]+/)+(?P<id>[0-9a-z]+)'
+    IE_NAME = 'onet.pl'
+
+    _TESTS = [{
+        'url': 'http://eurosport.onet.pl/zimowe/skoki-narciarskie/ziobro-wygral-kwalifikacje-w-pjongczangu/9ckrly',
+        'md5': 'b94021eb56214c3969380388b6e73cb0',
+        'info_dict': {
+            'id': '1561707.1685479',
+            'ext': 'mp4',
+            'title': 'Ziobro wygrał kwalifikacje w Pjongczangu',
+            'description': 'md5:61fb0740084d2d702ea96512a03585b4',
+            'upload_date': '20170214',
+            'timestamp': 1487078046,
+        },
+    }, {
+        # embedded via pulsembed
+        'url': 'http://film.onet.pl/pensjonat-nad-rozlewiskiem-relacja-z-planu-serialu/y428n0',
+        'info_dict': {
+            'id': '501235.965429946',
+            'ext': 'mp4',
+            'title': '"Pensjonat nad rozlewiskiem": relacja z planu serialu',
+            'upload_date': '20170622',
+            'timestamp': 1498159955,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://film.onet.pl/zwiastuny/ghost-in-the-shell-drugi-zwiastun-pl/5q6yl3',
+        'only_matching': True,
+    }, {
+        'url': 'http://moto.onet.pl/jak-wybierane-sa-miejsca-na-fotoradary/6rs04e',
+        'only_matching': True,
+    }, {
+        'url': 'http://businessinsider.com.pl/wideo/scenariusz-na-koniec-swiata-wedlug-nasa/dwnqptk',
+        'only_matching': True,
+    }, {
+        'url': 'http://plejada.pl/weronika-rosati-o-swoim-domniemanym-slubie/n2bq89',
+        'only_matching': True,
+    }]
+
+    def _search_mvp_id(self, webpage, default=NO_DEFAULT):
+        return self._search_regex(
+            r'data-(?:params-)?mvp=["\'](\d+\.\d+)', webpage, 'mvp id',
+            default=default)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        mvp_id = self._search_mvp_id(webpage, default=None)
+
+        if not mvp_id:
+            pulsembed_url = self._search_regex(
+                r'data-src=(["\'])(?P<url>(?:https?:)?//pulsembed\.eu/.+?)\1',
+                webpage, 'pulsembed url', group='url')
+            webpage = self._download_webpage(
+                pulsembed_url, video_id, 'Downloading pulsembed webpage')
+            mvp_id = self._search_mvp_id(webpage)
+
+        return self.url_result(
+            'onetmvp:%s' % mvp_id, OnetMVPIE.ie_key(), video_id=mvp_id)