Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/tvnow.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import compat_str
   8 from ..utils import (
   9     ExtractorError,
  10     int_or_none,
  11     parse_iso8601,
  12     parse_duration,
  13     str_or_none,
  14     update_url_query,
  15     urljoin,
  16 )
  17
  18
  19 class TVNowBaseIE(InfoExtractor):
  20     _VIDEO_FIELDS = (
  21         'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
  22         'broadcastStartDate', 'isDrm', 'duration', 'season', 'episode',
  23         'manifest.dashclear', 'manifest.hlsclear', 'manifest.smoothclear',
  24         'format.title', 'format.defaultImage169Format', 'format.defaultImage169Logo')
  25
  26     def _call_api(self, path, video_id, query):
  27         return self._download_json(
  28             'https://api.tvnow.de/v3/' + path, video_id, query=query)
  29
  30     def _extract_video(self, info, display_id):
  31         video_id = compat_str(info['id'])
  32         title = info['title']
  33
  34         paths = []
  35         for manifest_url in (info.get('manifest') or {}).values():
  36             if not manifest_url:
  37                 continue
  38             manifest_url = update_url_query(manifest_url, {'filter': ''})
  39             path = self._search_regex(r'https?://[^/]+/(.+?)\.ism/', manifest_url, 'path')
  40             if path in paths:
  41                 continue
  42             paths.append(path)
  43
  44             def url_repl(proto, suffix):
  45                 return re.sub(
  46                     r'(?:hls|dash|hss)([.-])', proto + r'\1', re.sub(
  47                         r'\.ism/(?:[^.]*\.(?:m3u8|mpd)|[Mm]anifest)',
  48                         '.ism/' + suffix, manifest_url))
  49
  50             def make_urls(proto, suffix):
  51                 urls = [url_repl(proto, suffix)]
  52                 hd_url = urls[0].replace('/manifest/', '/ngvod/')
  53                 if hd_url != urls[0]:
  54                     urls.append(hd_url)
  55                 return urls
  56
  57             for man_url in make_urls('dash', '.mpd'):
  58                 formats = self._extract_mpd_formats(
  59                     man_url, video_id, mpd_id='dash', fatal=False)
  60             for man_url in make_urls('hss', 'Manifest'):
  61                 formats.extend(self._extract_ism_formats(
  62                     man_url, video_id, ism_id='mss', fatal=False))
  63             for man_url in make_urls('hls', '.m3u8'):
  64                 formats.extend(self._extract_m3u8_formats(
  65                     man_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls',
  66                     fatal=False))
  67             if formats:
  68                 break
  69         else:
  70             if info.get('isDrm'):
  71                 raise ExtractorError(
  72                     'Video %s is DRM protected' % video_id, expected=True)
  73             if info.get('geoblocked'):
  74                 raise self.raise_geo_restricted()
  75             if not info.get('free', True):
  76                 raise ExtractorError(
  77                     'Video %s is not available for free' % video_id, expected=True)
  78         self._sort_formats(formats)
  79
  80         description = info.get('articleLong') or info.get('articleShort')
  81         timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
  82         duration = parse_duration(info.get('duration'))
  83
  84         f = info.get('format', {})
  85
  86         thumbnails = [{
  87             'url': 'https://aistvnow-a.akamaihd.net/tvnow/movie/%s' % video_id,
  88         }]
  89         thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
  90         if thumbnail:
  91             thumbnails.append({
  92                 'url': thumbnail,
  93             })
  94
  95         return {
  96             'id': video_id,
  97             'display_id': display_id,
  98             'title': title,
  99             'description': description,
 100             'thumbnails': thumbnails,
 101             'timestamp': timestamp,
 102             'duration': duration,
 103             'series': f.get('title'),
 104             'season_number': int_or_none(info.get('season')),
 105             'episode_number': int_or_none(info.get('episode')),
 106             'episode': title,
 107             'formats': formats,
 108         }
 109
 110
 111 class TVNowIE(TVNowBaseIE):
 112     _VALID_URL = r'''(?x)
 113                     https?://
 114                         (?:www\.)?tvnow\.(?:de|at|ch)/(?P<station>[^/]+)/
 115                         (?P<show_id>[^/]+)/
 116                         (?!(?:list|jahr)(?:/|$))(?P<id>[^/?\#&]+)
 117                     '''
 118
 119     @classmethod
 120     def suitable(cls, url):
 121         return (False if TVNowNewIE.suitable(url) or TVNowSeasonIE.suitable(url) or TVNowAnnualIE.suitable(url) or TVNowShowIE.suitable(url)
 122                 else super(TVNowIE, cls).suitable(url))
 123
 124     _TESTS = [{
 125         'url': 'https://www.tvnow.de/rtl2/grip-das-motormagazin/der-neue-porsche-911-gt-3/player',
 126         'info_dict': {
 127             'id': '331082',
 128             'display_id': 'grip-das-motormagazin/der-neue-porsche-911-gt-3',
 129             'ext': 'mp4',
 130             'title': 'Der neue Porsche 911 GT 3',
 131             'description': 'md5:6143220c661f9b0aae73b245e5d898bb',
 132             'timestamp': 1495994400,
 133             'upload_date': '20170528',
 134             'duration': 5283,
 135             'series': 'GRIP - Das Motormagazin',
 136             'season_number': 14,
 137             'episode_number': 405,
 138             'episode': 'Der neue Porsche 911 GT 3',
 139         },
 140     }, {
 141         # rtl2
 142         'url': 'https://www.tvnow.de/rtl2/armes-deutschland/episode-0008/player',
 143         'only_matching': True,
 144     }, {
 145         # rtlnitro
 146         'url': 'https://www.tvnow.de/nitro/alarm-fuer-cobra-11-die-autobahnpolizei/auf-eigene-faust-pilot/player',
 147         'only_matching': True,
 148     }, {
 149         # superrtl
 150         'url': 'https://www.tvnow.de/superrtl/die-lustigsten-schlamassel-der-welt/u-a-ketchup-effekt/player',
 151         'only_matching': True,
 152     }, {
 153         # ntv
 154         'url': 'https://www.tvnow.de/ntv/startup-news/goetter-in-weiss/player',
 155         'only_matching': True,
 156     }, {
 157         # vox
 158         'url': 'https://www.tvnow.de/vox/auto-mobil/neues-vom-automobilmarkt-2017-11-19-17-00-00/player',
 159         'only_matching': True,
 160     }, {
 161         # rtlplus
 162         'url': 'https://www.tvnow.de/rtlplus/op-ruft-dr-bruckner/die-vernaehte-frau/player',
 163         'only_matching': True,
 164     }, {
 165         'url': 'https://www.tvnow.de/rtl2/grip-das-motormagazin/der-neue-porsche-911-gt-3',
 166         'only_matching': True,
 167     }]
 168
 169     def _real_extract(self, url):
 170         mobj = re.match(self._VALID_URL, url)
 171         display_id = '%s/%s' % mobj.group(2, 3)
 172
 173         info = self._call_api(
 174             'movies/' + display_id, display_id, query={
 175                 'fields': ','.join(self._VIDEO_FIELDS),
 176             })
 177
 178         return self._extract_video(info, display_id)
 179
 180
 181 class TVNowNewIE(InfoExtractor):
 182     _VALID_URL = r'''(?x)
 183                     (?P<base_url>https?://
 184                         (?:www\.)?tvnow\.(?:de|at|ch)/
 185                         (?:shows|serien))/
 186                         (?P<show>[^/]+)-\d+/
 187                         [^/]+/
 188                         episode-\d+-(?P<episode>[^/?$&]+)-(?P<id>\d+)
 189                     '''
 190
 191     _TESTS = [{
 192         'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05/episode-405-der-neue-porsche-911-gt-3-331082',
 193         'only_matching': True,
 194     }]
 195
 196     def _real_extract(self, url):
 197         mobj = re.match(self._VALID_URL, url)
 198         base_url = re.sub(r'(?:shows|serien)', '_', mobj.group('base_url'))
 199         show, episode = mobj.group('show', 'episode')
 200         return self.url_result(
 201             # Rewrite new URLs to the old format and use extraction via old API
 202             # at api.tvnow.de as a loophole for bypassing premium content checks
 203             '%s/%s/%s' % (base_url, show, episode),
 204             ie=TVNowIE.ie_key(), video_id=mobj.group('id'))
 205
 206
 207 class TVNowNewBaseIE(InfoExtractor):
 208     def _call_api(self, path, video_id, query={}):
 209         result = self._download_json(
 210             'https://apigw.tvnow.de/module/' + path, video_id, query=query)
 211         error = result.get('error')
 212         if error:
 213             raise ExtractorError(
 214                 '%s said: %s' % (self.IE_NAME, error), expected=True)
 215         return result
 216
 217
 218 r"""
 219 TODO: new apigw.tvnow.de based version of TVNowIE. Replace old TVNowIE with it
 220 when api.tvnow.de is shut down. This version can't bypass premium checks though.
 221 class TVNowIE(TVNowNewBaseIE):
 222     _VALID_URL = r'''(?x)
 223                     https?://
 224                         (?:www\.)?tvnow\.(?:de|at|ch)/
 225                         (?:shows|serien)/[^/]+/
 226                         (?:[^/]+/)+
 227                         (?P<display_id>[^/?$&]+)-(?P<id>\d+)
 228                     '''
 229
 230     _TESTS = [{
 231         # episode with annual navigation
 232         'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05/episode-405-der-neue-porsche-911-gt-3-331082',
 233         'info_dict': {
 234             'id': '331082',
 235             'display_id': 'grip-das-motormagazin/der-neue-porsche-911-gt-3',
 236             'ext': 'mp4',
 237             'title': 'Der neue Porsche 911 GT 3',
 238             'description': 'md5:6143220c661f9b0aae73b245e5d898bb',
 239             'thumbnail': r're:^https?://.*\.jpg$',
 240             'timestamp': 1495994400,
 241             'upload_date': '20170528',
 242             'duration': 5283,
 243             'series': 'GRIP - Das Motormagazin',
 244             'season_number': 14,
 245             'episode_number': 405,
 246             'episode': 'Der neue Porsche 911 GT 3',
 247         },
 248     }, {
 249         # rtl2, episode with season navigation
 250         'url': 'https://www.tvnow.de/shows/armes-deutschland-11471/staffel-3/episode-14-bernd-steht-seit-der-trennung-von-seiner-frau-allein-da-526124',
 251         'only_matching': True,
 252     }, {
 253         # rtlnitro
 254         'url': 'https://www.tvnow.de/serien/alarm-fuer-cobra-11-die-autobahnpolizei-1815/staffel-13/episode-5-auf-eigene-faust-pilot-366822',
 255         'only_matching': True,
 256     }, {
 257         # superrtl
 258         'url': 'https://www.tvnow.de/shows/die-lustigsten-schlamassel-der-welt-1221/staffel-2/episode-14-u-a-ketchup-effekt-364120',
 259         'only_matching': True,
 260     }, {
 261         # ntv
 262         'url': 'https://www.tvnow.de/shows/startup-news-10674/staffel-2/episode-39-goetter-in-weiss-387630',
 263         'only_matching': True,
 264     }, {
 265         # vox
 266         'url': 'https://www.tvnow.de/shows/auto-mobil-174/2017-11/episode-46-neues-vom-automobilmarkt-2017-11-19-17-00-00-380072',
 267         'only_matching': True,
 268     }, {
 269         'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05/episode-405-der-neue-porsche-911-gt-3-331082',
 270         'only_matching': True,
 271     }]
 272
 273     def _extract_video(self, info, url, display_id):
 274         config = info['config']
 275         source = config['source']
 276
 277         video_id = compat_str(info.get('id') or source['videoId'])
 278         title = source['title'].strip()
 279
 280         paths = []
 281         for manifest_url in (info.get('manifest') or {}).values():
 282             if not manifest_url:
 283                 continue
 284             manifest_url = update_url_query(manifest_url, {'filter': ''})
 285             path = self._search_regex(r'https?://[^/]+/(.+?)\.ism/', manifest_url, 'path')
 286             if path in paths:
 287                 continue
 288             paths.append(path)
 289
 290             def url_repl(proto, suffix):
 291                 return re.sub(
 292                     r'(?:hls|dash|hss)([.-])', proto + r'\1', re.sub(
 293                         r'\.ism/(?:[^.]*\.(?:m3u8|mpd)|[Mm]anifest)',
 294                         '.ism/' + suffix, manifest_url))
 295
 296             formats = self._extract_mpd_formats(
 297                 url_repl('dash', '.mpd'), video_id,
 298                 mpd_id='dash', fatal=False)
 299             formats.extend(self._extract_ism_formats(
 300                 url_repl('hss', 'Manifest'),
 301                 video_id, ism_id='mss', fatal=False))
 302             formats.extend(self._extract_m3u8_formats(
 303                 url_repl('hls', '.m3u8'), video_id, 'mp4',
 304                 'm3u8_native', m3u8_id='hls', fatal=False))
 305             if formats:
 306                 break
 307         else:
 308             if try_get(info, lambda x: x['rights']['isDrm']):
 309                 raise ExtractorError(
 310                     'Video %s is DRM protected' % video_id, expected=True)
 311             if try_get(config, lambda x: x['boards']['geoBlocking']['block']):
 312                 raise self.raise_geo_restricted()
 313             if not info.get('free', True):
 314                 raise ExtractorError(
 315                     'Video %s is not available for free' % video_id, expected=True)
 316         self._sort_formats(formats)
 317
 318         description = source.get('description')
 319         thumbnail = url_or_none(source.get('poster'))
 320         timestamp = unified_timestamp(source.get('previewStart'))
 321         duration = parse_duration(source.get('length'))
 322
 323         series = source.get('format')
 324         season_number = int_or_none(self._search_regex(
 325             r'staffel-(\d+)', url, 'season number', default=None))
 326         episode_number = int_or_none(self._search_regex(
 327             r'episode-(\d+)', url, 'episode number', default=None))
 328
 329         return {
 330             'id': video_id,
 331             'display_id': display_id,
 332             'title': title,
 333             'description': description,
 334             'thumbnail': thumbnail,
 335             'timestamp': timestamp,
 336             'duration': duration,
 337             'series': series,
 338             'season_number': season_number,
 339             'episode_number': episode_number,
 340             'episode': title,
 341             'formats': formats,
 342         }
 343
 344     def _real_extract(self, url):
 345         display_id, video_id = re.match(self._VALID_URL, url).groups()
 346         info = self._call_api('player/' + video_id, video_id)
 347         return self._extract_video(info, video_id, display_id)
 348 """
 349
 350
 351 class TVNowListBaseIE(TVNowNewBaseIE):
 352     _SHOW_VALID_URL = r'''(?x)
 353                     (?P<base_url>
 354                         https?://
 355                             (?:www\.)?tvnow\.(?:de|at|ch)/(?:shows|serien)/
 356                             [^/?#&]+-(?P<show_id>\d+)
 357                     )
 358                     '''
 359
 360     @classmethod
 361     def suitable(cls, url):
 362         return (False if TVNowNewIE.suitable(url)
 363                 else super(TVNowListBaseIE, cls).suitable(url))
 364
 365     def _extract_items(self, url, show_id, list_id, query):
 366         items = self._call_api(
 367             'teaserrow/format/episode/' + show_id, list_id,
 368             query=query)['items']
 369
 370         entries = []
 371         for item in items:
 372             if not isinstance(item, dict):
 373                 continue
 374             item_url = urljoin(url, item.get('url'))
 375             if not item_url:
 376                 continue
 377             video_id = str_or_none(item.get('id') or item.get('videoId'))
 378             item_title = item.get('subheadline') or item.get('text')
 379             entries.append(self.url_result(
 380                 item_url, ie=TVNowNewIE.ie_key(), video_id=video_id,
 381                 video_title=item_title))
 382
 383         return self.playlist_result(entries, '%s/%s' % (show_id, list_id))
 384
 385
 386 class TVNowSeasonIE(TVNowListBaseIE):
 387     _VALID_URL = r'%s/staffel-(?P<id>\d+)' % TVNowListBaseIE._SHOW_VALID_URL
 388     _TESTS = [{
 389         'url': 'https://www.tvnow.de/serien/alarm-fuer-cobra-11-die-autobahnpolizei-1815/staffel-13',
 390         'info_dict': {
 391             'id': '1815/13',
 392         },
 393         'playlist_mincount': 22,
 394     }]
 395
 396     def _real_extract(self, url):
 397         _, show_id, season_id = re.match(self._VALID_URL, url).groups()
 398         return self._extract_items(
 399             url, show_id, season_id, {'season': season_id})
 400
 401
 402 class TVNowAnnualIE(TVNowListBaseIE):
 403     _VALID_URL = r'%s/(?P<year>\d{4})-(?P<month>\d{2})' % TVNowListBaseIE._SHOW_VALID_URL
 404     _TESTS = [{
 405         'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669/2017-05',
 406         'info_dict': {
 407             'id': '1669/2017-05',
 408         },
 409         'playlist_mincount': 2,
 410     }]
 411
 412     def _real_extract(self, url):
 413         _, show_id, year, month = re.match(self._VALID_URL, url).groups()
 414         return self._extract_items(
 415             url, show_id, '%s-%s' % (year, month), {
 416                 'year': int(year),
 417                 'month': int(month),
 418             })
 419
 420
 421 class TVNowShowIE(TVNowListBaseIE):
 422     _VALID_URL = TVNowListBaseIE._SHOW_VALID_URL
 423     _TESTS = [{
 424         # annual navigationType
 425         'url': 'https://www.tvnow.de/shows/grip-das-motormagazin-1669',
 426         'info_dict': {
 427             'id': '1669',
 428         },
 429         'playlist_mincount': 73,
 430     }, {
 431         # season navigationType
 432         'url': 'https://www.tvnow.de/shows/armes-deutschland-11471',
 433         'info_dict': {
 434             'id': '11471',
 435         },
 436         'playlist_mincount': 3,
 437     }]
 438
 439     @classmethod
 440     def suitable(cls, url):
 441         return (False if TVNowNewIE.suitable(url) or TVNowSeasonIE.suitable(url) or TVNowAnnualIE.suitable(url)
 442                 else super(TVNowShowIE, cls).suitable(url))
 443
 444     def _real_extract(self, url):
 445         base_url, show_id = re.match(self._VALID_URL, url).groups()
 446
 447         result = self._call_api(
 448             'teaserrow/format/navigation/' + show_id, show_id)
 449
 450         items = result['items']
 451
 452         entries = []
 453         navigation = result.get('navigationType')
 454         if navigation == 'annual':
 455             for item in items:
 456                 if not isinstance(item, dict):
 457                     continue
 458                 year = int_or_none(item.get('year'))
 459                 if year is None:
 460                     continue
 461                 months = item.get('months')
 462                 if not isinstance(months, list):
 463                     continue
 464                 for month_dict in months:
 465                     if not isinstance(month_dict, dict) or not month_dict:
 466                         continue
 467                     month_number = int_or_none(list(month_dict.keys())[0])
 468                     if month_number is None:
 469                         continue
 470                     entries.append(self.url_result(
 471                         '%s/%04d-%02d' % (base_url, year, month_number),
 472                         ie=TVNowAnnualIE.ie_key()))
 473         elif navigation == 'season':
 474             for item in items:
 475                 if not isinstance(item, dict):
 476                     continue
 477                 season_number = int_or_none(item.get('season'))
 478                 if season_number is None:
 479                     continue
 480                 entries.append(self.url_result(
 481                     '%s/staffel-%d' % (base_url, season_number),
 482                     ie=TVNowSeasonIE.ie_key()))
 483         else:
 484             raise ExtractorError('Unknown navigationType')
 485
 486         return self.playlist_result(entries, show_id)