Set upstream metadata fields: Bug-Database, Bug-Submit, Repository, Repository-Browse.

[youtubedl] / youtube_dl / extractor / ndr.py

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 16213eed90119266207bd1936adf098050a92c4b..2447c812e021e73991082aefab4bd98e6dd000a1 100644 (file)
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -7,8 +7,11 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     int_or_none,
 from ..utils import (
     determine_ext,
     int_or_none,

+    merge_dicts,

     parse_iso8601,
     qualities,
     parse_iso8601,
     qualities,

+    try_get,
+    urljoin,

 )
 
 
 )
 
 


@@ -23,7 +26,7 @@ class NDRBaseIE(InfoExtractor):
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
     IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
     IE_DESC = 'NDR.de - Norddeutscher Rundfunk'

-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
+    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'

     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',


@@ -85,27 +88,31 @@ class NDRIE(NDRBaseIE):
 
     def _extract_embed(self, webpage, display_id):
         embed_url = self._html_search_meta(
 
     def _extract_embed(self, webpage, display_id):
         embed_url = self._html_search_meta(

-            'embedURL', webpage, 'embed URL', fatal=True)
+            'embedURL', webpage, 'embed URL',
+            default=None) or self._search_regex(
+            r'\bembedUrl["\']\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            'embed URL', group='url')

         description = self._search_regex(
             r'<p[^>]+itemprop="description">([^<]+)</p>',
         description = self._search_regex(
             r'<p[^>]+itemprop="description">([^<]+)</p>',

-            webpage, 'description', fatal=False)
+            webpage, 'description', default=None) or self._og_search_description(webpage)

         timestamp = parse_iso8601(
             self._search_regex(
         timestamp = parse_iso8601(
             self._search_regex(

-                r'<span itemprop="datePublished" content="([^"]+)">',
-                webpage, 'upload date', fatal=False))
-        return {
+                r'<span[^>]+itemprop="(?:datePublished|uploadDate)"[^>]+content="([^"]+)"',
+                webpage, 'upload date', default=None))
+        info = self._search_json_ld(webpage, display_id, default={})
+        return merge_dicts({

             '_type': 'url_transparent',
             'url': embed_url,
             'display_id': display_id,
             'description': description,
             'timestamp': timestamp,
             '_type': 'url_transparent',
             'url': embed_url,
             'display_id': display_id,
             'description': description,
             'timestamp': timestamp,

-        }
+        }, info)

 
 
 class NJoyIE(NDRBaseIE):
     IE_NAME = 'njoy'
     IE_DESC = 'N-JOY'
 
 
 class NJoyIE(NDRBaseIE):
     IE_NAME = 'njoy'
     IE_DESC = 'N-JOY'

-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'

     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',


@@ -190,10 +197,12 @@ class NDREmbedBaseIE(InfoExtractor):
             ext = determine_ext(src, None)
             if ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
             ext = determine_ext(src, None)
             if ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(

-                    src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id, f4m_id='hds'))
+                    src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id,
+                    f4m_id='hds', fatal=False))

             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
             elif ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(

-                    src, video_id, m3u8_id='hls', entry_protocol='m3u8_native'))
+                    src, video_id, 'mp4', m3u8_id='hls',
+                    entry_protocol='m3u8_native', fatal=False))

             else:
                 quality = f.get('quality')
                 ff = {
             else:
                 quality = f.get('quality')
                 ff = {


@@ -218,11 +227,17 @@ class NDREmbedBaseIE(InfoExtractor):
         upload_date = ppjson.get('config', {}).get('publicationDate')
         duration = int_or_none(config.get('duration'))
 
         upload_date = ppjson.get('config', {}).get('publicationDate')
         duration = int_or_none(config.get('duration'))
 

-        thumbnails = [{
-            'id': thumbnail.get('quality') or thumbnail_id,
-            'url': thumbnail['src'],
-            'preference': quality_key(thumbnail.get('quality')),
-        } for thumbnail_id, thumbnail in config.get('poster', {}).items() if thumbnail.get('src')]
+        thumbnails = []
+        poster = try_get(config, lambda x: x['poster'], dict) or {}
+        for thumbnail_id, thumbnail in poster.items():
+            thumbnail_url = urljoin(url, thumbnail.get('src'))
+            if not thumbnail_url:
+                continue
+            thumbnails.append({
+                'id': thumbnail.get('quality') or thumbnail_id,
+                'url': thumbnail_url,
+                'preference': quality_key(thumbnail.get('quality')),
+            })

 
         return {
             'id': video_id,
 
         return {
             'id': video_id,


@@ -238,7 +253,7 @@ class NDREmbedBaseIE(InfoExtractor):
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'

-    _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
+    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'

     _TESTS = [{
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
         'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',
     _TESTS = [{
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
         'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',


@@ -302,7 +317,7 @@ class NDREmbedIE(NDREmbedBaseIE):
         'info_dict': {
             'id': 'livestream217',
             'ext': 'flv',
         'info_dict': {
             'id': 'livestream217',
             'ext': 'flv',

-            'title': 're:^NDR Fernsehen Niedersachsen \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'title': r're:^NDR Fernsehen Niedersachsen \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',

             'is_live': True,
             'upload_date': '20150910',
         },
             'is_live': True,
             'upload_date': '20150910',
         },


@@ -332,7 +347,7 @@ class NDREmbedIE(NDREmbedBaseIE):
 
 class NJoyEmbedIE(NDREmbedBaseIE):
     IE_NAME = 'njoy:embed'
 
 class NJoyEmbedIE(NDREmbedBaseIE):
     IE_NAME = 'njoy:embed'

-    _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
+    _VALID_URL = r'https?://(?:www\.)?n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'

     _TESTS = [{
         # httpVideo
         'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',
     _TESTS = [{
         # httpVideo
         'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',


@@ -367,7 +382,7 @@ class NJoyEmbedIE(NDREmbedBaseIE):
         'info_dict': {
             'id': 'webradioweltweit100',
             'ext': 'mp3',
         'info_dict': {
             'id': 'webradioweltweit100',
             'ext': 'mp3',

-            'title': 're:^N-JOY Weltweit \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'title': r're:^N-JOY Weltweit \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',

             'is_live': True,
             'uploader': 'njoy',
             'upload_date': '20150810',
             'is_live': True,
             'uploader': 'njoy',
             'upload_date': '20150810',