Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ard.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from .generic import GenericIE
   8 from ..utils import (
   9     determine_ext,
  10     ExtractorError,
  11     qualities,
  12     int_or_none,
  13     parse_duration,
  14     unified_strdate,
  15     xpath_text,
  16     parse_xml,
  17 )
  18
  19
  20 class ARDMediathekIE(InfoExtractor):
  21     IE_NAME = 'ARD:mediathek'
  22     _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?'
  23
  24     _TESTS = [{
  25         'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
  26         'only_matching': True,
  27     }, {
  28         'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
  29         'info_dict': {
  30             'id': '22490580',
  31             'ext': 'mp4',
  32             'title': 'Das Wunder von Wolbeck (Video tgl. ab 20 Uhr)',
  33             'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.',
  34         },
  35         'skip': 'Blocked outside of Germany',
  36     }]
  37
  38     def _real_extract(self, url):
  39         # determine video id from url
  40         m = re.match(self._VALID_URL, url)
  41
  42         numid = re.search(r'documentId=([0-9]+)', url)
  43         if numid:
  44             video_id = numid.group(1)
  45         else:
  46             video_id = m.group('video_id')
  47
  48         webpage = self._download_webpage(url, video_id)
  49
  50         if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
  51             raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
  52
  53         if re.search(r'[\?&]rss($|[=&])', url):
  54             doc = parse_xml(webpage)
  55             if doc.tag == 'rss':
  56                 return GenericIE()._extract_rss(url, video_id, doc)
  57
  58         title = self._html_search_regex(
  59             [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
  60              r'<meta name="dcterms.title" content="(.*?)"/>',
  61              r'<h4 class="headline">(.*?)</h4>'],
  62             webpage, 'title')
  63         description = self._html_search_meta(
  64             'dcterms.abstract', webpage, 'description', default=None)
  65         if description is None:
  66             description = self._html_search_meta(
  67                 'description', webpage, 'meta description')
  68
  69         # Thumbnail is sometimes not present.
  70         # It is in the mobile version, but that seems to use a different URL
  71         # structure altogether.
  72         thumbnail = self._og_search_thumbnail(webpage, default=None)
  73
  74         media_streams = re.findall(r'''(?x)
  75             mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s*
  76             "([^"]+)"''', webpage)
  77
  78         if media_streams:
  79             QUALITIES = qualities(['lo', 'hi', 'hq'])
  80             formats = []
  81             for furl in set(media_streams):
  82                 if furl.endswith('.f4m'):
  83                     fid = 'f4m'
  84                 else:
  85                     fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl)
  86                     fid = fid_m.group(1) if fid_m else None
  87                 formats.append({
  88                     'quality': QUALITIES(fid),
  89                     'format_id': fid,
  90                     'url': furl,
  91                 })
  92         else:  # request JSON file
  93             media_info = self._download_json(
  94                 'http://www.ardmediathek.de/play/media/%s' % video_id, video_id)
  95             # The second element of the _mediaArray contains the standard http urls
  96             streams = media_info['_mediaArray'][1]['_mediaStreamArray']
  97             if not streams:
  98                 if '"fsk"' in webpage:
  99                     raise ExtractorError('This video is only available after 20:00')
 100
 101             formats = []
 102             for s in streams:
 103                 if type(s['_stream']) == list:
 104                     for index, url in enumerate(s['_stream'][::-1]):
 105                         quality = s['_quality'] + index
 106                         formats.append({
 107                             'quality': quality,
 108                             'url': url,
 109                             'format_id': '%s-%s' % (determine_ext(url), quality)
 110                         })
 111                     continue
 112
 113                 format = {
 114                     'quality': s['_quality'],
 115                     'url': s['_stream'],
 116                 }
 117
 118                 format['format_id'] = '%s-%s' % (
 119                     determine_ext(format['url']), format['quality'])
 120
 121                 formats.append(format)
 122
 123         self._sort_formats(formats)
 124
 125         return {
 126             'id': video_id,
 127             'title': title,
 128             'description': description,
 129             'formats': formats,
 130             'thumbnail': thumbnail,
 131         }
 132
 133
 134 class ARDIE(InfoExtractor):
 135     _VALID_URL = '(?P<mainurl>https?://(www\.)?daserste\.de/[^?#]+/videos/(?P<display_id>[^/?#]+)-(?P<id>[0-9]+))\.html'
 136     _TEST = {
 137         'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
 138         'md5': 'd216c3a86493f9322545e045ddc3eb35',
 139         'info_dict': {
 140             'display_id': 'die-story-im-ersten-mission-unter-falscher-flagge',
 141             'id': '100',
 142             'ext': 'mp4',
 143             'duration': 2600,
 144             'title': 'Die Story im Ersten: Mission unter falscher Flagge',
 145             'upload_date': '20140804',
 146             'thumbnail': 're:^https?://.*\.jpg$',
 147         }
 148     }
 149
 150     def _real_extract(self, url):
 151         mobj = re.match(self._VALID_URL, url)
 152         display_id = mobj.group('display_id')
 153
 154         player_url = mobj.group('mainurl') + '~playerXml.xml'
 155         doc = self._download_xml(player_url, display_id)
 156         video_node = doc.find('./video')
 157         upload_date = unified_strdate(xpath_text(
 158             video_node, './broadcastDate'))
 159         thumbnail = xpath_text(video_node, './/teaserImage//variant/url')
 160
 161         formats = []
 162         for a in video_node.findall('.//asset'):
 163             f = {
 164                 'format_id': a.attrib['type'],
 165                 'width': int_or_none(a.find('./frameWidth').text),
 166                 'height': int_or_none(a.find('./frameHeight').text),
 167                 'vbr': int_or_none(a.find('./bitrateVideo').text),
 168                 'abr': int_or_none(a.find('./bitrateAudio').text),
 169                 'vcodec': a.find('./codecVideo').text,
 170                 'tbr': int_or_none(a.find('./totalBitrate').text),
 171             }
 172             if a.find('./serverPrefix').text:
 173                 f['url'] = a.find('./serverPrefix').text
 174                 f['playpath'] = a.find('./fileName').text
 175             else:
 176                 f['url'] = a.find('./fileName').text
 177             formats.append(f)
 178         self._sort_formats(formats)
 179
 180         return {
 181             'id': mobj.group('id'),
 182             'formats': formats,
 183             'display_id': display_id,
 184             'title': video_node.find('./title').text,
 185             'duration': parse_duration(video_node.find('./duration').text),
 186             'upload_date': upload_date,
 187             'thumbnail': thumbnail,
 188         }