Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/rtlnl.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 from .common import InfoExtractor
   5 from ..utils import (
   6     int_or_none,
   7     parse_duration,
   8 )
   9
  10
  11 class RtlNlIE(InfoExtractor):
  12     IE_NAME = 'rtl.nl'
  13     IE_DESC = 'rtl.nl and rtlxl.nl'
  14     _VALID_URL = r'''(?x)
  15         https?://(?:(?:www|static)\.)?
  16         (?:
  17             rtlxl\.nl/[^\#]*\#!/[^/]+/|
  18             rtl\.nl/(?:(?:system/videoplayer/(?:[^/]+/)+(?:video_)?embed\.html|embed)\b.+?\buuid=|video/)
  19         )
  20         (?P<id>[0-9a-f-]+)'''
  21
  22     _TESTS = [{
  23         'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/82b1aad1-4a14-3d7b-b554-b0aed1b2c416',
  24         'md5': '473d1946c1fdd050b2c0161a4b13c373',
  25         'info_dict': {
  26             'id': '82b1aad1-4a14-3d7b-b554-b0aed1b2c416',
  27             'ext': 'mp4',
  28             'title': 'RTL Nieuws',
  29             'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
  30             'timestamp': 1461951000,
  31             'upload_date': '20160429',
  32             'duration': 1167.96,
  33         },
  34     }, {
  35         # best format avaialble a3t
  36         'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed/autoplay=false',
  37         'md5': 'dea7474214af1271d91ef332fb8be7ea',
  38         'info_dict': {
  39             'id': '84ae5571-ac25-4225-ae0c-ef8d9efb2aed',
  40             'ext': 'mp4',
  41             'timestamp': 1424039400,
  42             'title': 'RTL Nieuws - Nieuwe beelden Kopenhagen: chaos direct na aanslag',
  43             'thumbnail': r're:^https?://screenshots\.rtl\.nl/(?:[^/]+/)*sz=[0-9]+x[0-9]+/uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed$',
  44             'upload_date': '20150215',
  45             'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
  46         }
  47     }, {
  48         # empty synopsis and missing episodes (see https://github.com/rg3/youtube-dl/issues/6275)
  49         # best format available nettv
  50         'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a/autoplay=false',
  51         'info_dict': {
  52             'id': 'f536aac0-1dc3-4314-920e-3bd1c5b3811a',
  53             'ext': 'mp4',
  54             'title': 'RTL Nieuws - Meer beelden van overval juwelier',
  55             'thumbnail': r're:^https?://screenshots\.rtl\.nl/(?:[^/]+/)*sz=[0-9]+x[0-9]+/uuid=f536aac0-1dc3-4314-920e-3bd1c5b3811a$',
  56             'timestamp': 1437233400,
  57             'upload_date': '20150718',
  58             'duration': 30.474,
  59         },
  60         'params': {
  61             'skip_download': True,
  62         },
  63     }, {
  64         # encrypted m3u8 streams, georestricted
  65         'url': 'http://www.rtlxl.nl/#!/afl-2-257632/52a74543-c504-4cde-8aa8-ec66fe8d68a7',
  66         'only_matching': True,
  67     }, {
  68         'url': 'http://www.rtl.nl/system/videoplayer/derden/embed.html#!/uuid=bb0353b0-d6a4-1dad-90e9-18fe75b8d1f0',
  69         'only_matching': True,
  70     }, {
  71         'url': 'http://rtlxl.nl/?_ga=1.204735956.572365465.1466978370#!/rtl-nieuws-132237/3c487912-023b-49ac-903e-2c5d79f8410f',
  72         'only_matching': True,
  73     }, {
  74         'url': 'https://www.rtl.nl/video/c603c9c2-601d-4b5e-8175-64f1e942dc7d/',
  75         'only_matching': True,
  76     }, {
  77         'url': 'https://static.rtl.nl/embed/?uuid=1a2970fc-5c0b-43ff-9fdc-927e39e6d1bc&autoplay=false&publicatiepunt=rtlnieuwsnl',
  78         'only_matching': True,
  79     }]
  80
  81     def _real_extract(self, url):
  82         uuid = self._match_id(url)
  83         info = self._download_json(
  84             'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=adaptive/' % uuid,
  85             uuid)
  86
  87         material = info['material'][0]
  88         title = info['abstracts'][0]['name']
  89         subtitle = material.get('title')
  90         if subtitle:
  91             title += ' - %s' % subtitle
  92         description = material.get('synopsis')
  93
  94         meta = info.get('meta', {})
  95
  96         # m3u8 streams are encrypted and may not be handled properly by older ffmpeg/avconv.
  97         # To workaround this previously adaptive -> flash trick was used to obtain
  98         # unencrypted m3u8 streams (see https://github.com/rg3/youtube-dl/issues/4118)
  99         # and bypass georestrictions as well.
 100         # Currently, unencrypted m3u8 playlists are (intentionally?) invalid and therefore
 101         # unusable albeit can be fixed by simple string replacement (see
 102         # https://github.com/rg3/youtube-dl/pull/6337)
 103         # Since recent ffmpeg and avconv handle encrypted streams just fine encrypted
 104         # streams are used now.
 105         videopath = material['videopath']
 106         m3u8_url = meta.get('videohost', 'http://manifest.us.rtl.nl') + videopath
 107
 108         formats = self._extract_m3u8_formats(
 109             m3u8_url, uuid, 'mp4', m3u8_id='hls', fatal=False)
 110
 111         video_urlpart = videopath.split('/adaptive/')[1][:-5]
 112         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 113
 114         PG_FORMATS = (
 115             ('a2t', 512, 288),
 116             ('a3t', 704, 400),
 117             ('nettv', 1280, 720),
 118         )
 119
 120         def pg_format(format_id, width, height):
 121             return {
 122                 'url': PG_URL_TEMPLATE % (format_id, video_urlpart),
 123                 'format_id': 'pg-%s' % format_id,
 124                 'protocol': 'http',
 125                 'width': width,
 126                 'height': height,
 127             }
 128
 129         if not formats:
 130             formats = [pg_format(*pg_tuple) for pg_tuple in PG_FORMATS]
 131         else:
 132             pg_formats = []
 133             for format_id, width, height in PG_FORMATS:
 134                 try:
 135                     # Find hls format with the same width and height corresponding
 136                     # to progressive format and copy metadata from it.
 137                     f = next(f for f in formats if f.get('height') == height)
 138                     # hls formats may have invalid width
 139                     f['width'] = width
 140                     f_copy = f.copy()
 141                     f_copy.update(pg_format(format_id, width, height))
 142                     pg_formats.append(f_copy)
 143                 except StopIteration:
 144                     # Missing hls format does mean that no progressive format with
 145                     # such width and height exists either.
 146                     pass
 147             formats.extend(pg_formats)
 148         self._sort_formats(formats)
 149
 150         thumbnails = []
 151
 152         for p in ('poster_base_url', '"thumb_base_url"'):
 153             if not meta.get(p):
 154                 continue
 155
 156             thumbnails.append({
 157                 'url': self._proto_relative_url(meta[p] + uuid),
 158                 'width': int_or_none(self._search_regex(
 159                     r'/sz=([0-9]+)', meta[p], 'thumbnail width', fatal=False)),
 160                 'height': int_or_none(self._search_regex(
 161                     r'/sz=[0-9]+x([0-9]+)',
 162                     meta[p], 'thumbnail height', fatal=False))
 163             })
 164
 165         return {
 166             'id': uuid,
 167             'title': title,
 168             'formats': formats,
 169             'timestamp': material['original_date'],
 170             'description': description,
 171             'duration': parse_duration(material.get('duration')),
 172             'thumbnails': thumbnails,
 173         }