Update upstream source from tag 'upstream/2019.06.08'

[youtubedl] / youtube_dl / extractor / ccc.py

diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
index 2a5d4be185d64f47ce37ca83f9bb6bd1d7eac829..36e6dff72c9f4bf62b59b864091a7b3a0d17ba5b 100644 (file)
--- a/youtube_dl/extractor/ccc.py
+++ b/youtube_dl/extractor/ccc.py
@@ -1,99 +1,111 @@
+# coding: utf-8

 from __future__ import unicode_literals
 
 from __future__ import unicode_literals
 

-import re
-

 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,

-    qualities,
-    unified_strdate,
+    parse_iso8601,
+    try_get,
+    url_or_none,

 )
 
 
 class CCCIE(InfoExtractor):
     IE_NAME = 'media.ccc.de'
 )
 
 
 class CCCIE(InfoExtractor):
     IE_NAME = 'media.ccc.de'

-    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/[^?#]+/[^?#/]*?_(?P<id>[0-9]{8,})._[^?#/]*\.html'
+    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/v/(?P<id>[^/?#&]+)'

 
 

-    _TEST = {
-        'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
-        'md5': '205a365d0d57c0b1e43a12c9ffe8f9be',
+    _TESTS = [{
+        'url': 'https://media.ccc.de/v/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor#video',
+        'md5': '3a1eda8f3a29515d27f5adb967d7e740',

         'info_dict': {
         'info_dict': {

-            'id': '20131228183',
+            'id': '1839',

             'ext': 'mp4',
             'title': 'Introduction to Processor Design',
             'ext': 'mp4',
             'title': 'Introduction to Processor Design',

-            'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'view_count': int,
-            'upload_date': '20131229',
+            'creator': 'byterazor',
+            'description': 'md5:df55f6d073d4ceae55aae6f2fd98a0ac',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'upload_date': '20131228',
+            'timestamp': 1388188800,
+            'duration': 3710,
+            'tags': list,

         }
         }

-    }
+    }, {
+        'url': 'https://media.ccc.de/v/32c3-7368-shopshifting#download',
+        'only_matching': True,
+    }]

 
     def _real_extract(self, url):
 
     def _real_extract(self, url):

-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        if self._downloader.params.get('prefer_free_formats'):
-            preference = qualities(['mp3', 'opus', 'mp4-lq', 'webm-lq', 'h264-sd', 'mp4-sd', 'webm-sd', 'mp4', 'webm', 'mp4-hd', 'h264-hd', 'webm-hd'])
-        else:
-            preference = qualities(['opus', 'mp3', 'webm-lq', 'mp4-lq', 'webm-sd', 'h264-sd', 'mp4-sd', 'webm', 'mp4', 'webm-hd', 'mp4-hd', 'h264-hd'])
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        event_id = self._search_regex(r"data-id='(\d+)'", webpage, 'event id')
+        event_data = self._download_json('https://media.ccc.de/public/events/%s' % event_id, event_id)

 
 

-        title = self._html_search_regex(
-            r'(?s)<h1>(.*?)</h1>', webpage, 'title')
-        description = self._html_search_regex(
-            r"(?s)<p class='description'>(.*?)</p>",
-            webpage, 'description', fatal=False)
-        upload_date = unified_strdate(self._html_search_regex(
-            r"(?s)<span class='[^']*fa-calendar-o'></span>(.*?)</li>",
-            webpage, 'upload date', fatal=False))
-        view_count = int_or_none(self._html_search_regex(
-            r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",
-            webpage, 'view count', fatal=False))
-
-        matches = re.finditer(r'''(?xs)
-            <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
-            <a\s+href='(?P<http_url>[^']+)'>\s*
-            (?:
-                .*?
-                <a\s+href='(?P<torrent_url>[^']+\.torrent)'
-            )?''', webpage)

         formats = []
         formats = []

-        for m in matches:
-            format = m.group('format')
-            format_id = self._search_regex(
-                r'.*/([a-z0-9_-]+)/[^/]*$',
-                m.group('http_url'), 'format id', default=None)
-            vcodec = 'h264' if 'h264' in format_id else (
-                'none' if format_id in ('mp3', 'opus') else None
+        for recording in event_data.get('recordings', []):
+            recording_url = recording.get('recording_url')
+            if not recording_url:
+                continue
+            language = recording.get('language')
+            folder = recording.get('folder')
+            format_id = None
+            if language:
+                format_id = language
+            if folder:
+                if language:
+                    format_id += '-' + folder
+                else:
+                    format_id = folder
+            vcodec = 'h264' if 'h264' in folder else (
+                'none' if folder in ('mp3', 'opus') else None

             )
             formats.append({
                 'format_id': format_id,
             )
             formats.append({
                 'format_id': format_id,

-                'format': format,
-                'url': m.group('http_url'),
+                'url': recording_url,
+                'width': int_or_none(recording.get('width')),
+                'height': int_or_none(recording.get('height')),
+                'filesize': int_or_none(recording.get('size'), invscale=1024 * 1024),
+                'language': language,

                 'vcodec': vcodec,
                 'vcodec': vcodec,

-                'preference': preference(format_id),

             })
             })

-
-            if m.group('torrent_url'):
-                formats.append({
-                    'format_id': 'torrent-%s' % (format if format_id is None else format_id),
-                    'format': '%s (torrent)' % format,
-                    'proto': 'torrent',
-                    'format_note': '(unsupported; will just download the .torrent file)',
-                    'vcodec': vcodec,
-                    'preference': -100 + preference(format_id),
-                    'url': m.group('torrent_url'),
-                })

         self._sort_formats(formats)
 
         self._sort_formats(formats)
 

-        thumbnail = self._html_search_regex(
-            r"<video.*?poster='([^']+)'", webpage, 'thumbnail', fatal=False)
-

         return {
         return {

-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'view_count': view_count,
-            'upload_date': upload_date,
+            'id': event_id,
+            'display_id': display_id,
+            'title': event_data['title'],
+            'creator': try_get(event_data, lambda x: ', '.join(x['persons'])),
+            'description': event_data.get('description'),
+            'thumbnail': event_data.get('thumb_url'),
+            'timestamp': parse_iso8601(event_data.get('date')),
+            'duration': int_or_none(event_data.get('length')),
+            'tags': event_data.get('tags'),

             'formats': formats,
         }
             'formats': formats,
         }

+
+
+class CCCPlaylistIE(InfoExtractor):
+    IE_NAME = 'media.ccc.de:lists'
+    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/c/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://media.ccc.de/c/30c3',
+        'info_dict': {
+            'title': '30C3',
+            'id': '30c3',
+        },
+        'playlist_count': 135,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url).lower()
+
+        conf = self._download_json(
+            'https://media.ccc.de/public/conferences/' + playlist_id,
+            playlist_id)
+
+        entries = []
+        for e in conf['events']:
+            event_url = url_or_none(e.get('frontend_link'))
+            if event_url:
+                entries.append(self.url_result(event_url, ie=CCCIE.ie_key()))
+
+        return self.playlist_result(entries, playlist_id, conf.get('title'))