]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/webofstories.py
Update upstream source from tag 'upstream/2019.06.08'
[youtubedl] / youtube_dl / extractor / webofstories.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..utils import (
8 int_or_none,
9 orderedSet,
10 )
11
12
13 class WebOfStoriesIE(InfoExtractor):
14 _VALID_URL = r'https?://(?:www\.)?webofstories\.com/play/(?:[^/]+/)?(?P<id>[0-9]+)'
15 _VIDEO_DOMAIN = 'http://eu-mobile.webofstories.com/'
16 _GREAT_LIFE_STREAMER = 'rtmp://eu-cdn1.webofstories.com/cfx/st/'
17 _USER_STREAMER = 'rtmp://eu-users.webofstories.com/cfx/st/'
18 _TESTS = [{
19 'url': 'http://www.webofstories.com/play/hans.bethe/71',
20 'md5': '373e4dd915f60cfe3116322642ddf364',
21 'info_dict': {
22 'id': '4536',
23 'ext': 'mp4',
24 'title': 'The temperature of the sun',
25 'thumbnail': r're:^https?://.*\.jpg$',
26 'description': 'Hans Bethe talks about calculating the temperature of the sun',
27 'duration': 238,
28 }
29 }, {
30 'url': 'http://www.webofstories.com/play/55908',
31 'md5': '2985a698e1fe3211022422c4b5ed962c',
32 'info_dict': {
33 'id': '55908',
34 'ext': 'mp4',
35 'title': 'The story of Gemmata obscuriglobus',
36 'thumbnail': r're:^https?://.*\.jpg$',
37 'description': 'Planctomycete talks about The story of Gemmata obscuriglobus',
38 'duration': 169,
39 },
40 'skip': 'notfound',
41 }, {
42 # malformed og:title meta
43 'url': 'http://www.webofstories.com/play/54215?o=MS',
44 'info_dict': {
45 'id': '54215',
46 'ext': 'mp4',
47 'title': '"A Leg to Stand On"',
48 'thumbnail': r're:^https?://.*\.jpg$',
49 'description': 'Oliver Sacks talks about the death and resurrection of a limb',
50 'duration': 97,
51 },
52 'params': {
53 'skip_download': True,
54 },
55 }]
56
57 def _real_extract(self, url):
58 video_id = self._match_id(url)
59
60 webpage = self._download_webpage(url, video_id)
61 # Sometimes og:title meta is malformed
62 title = self._og_search_title(webpage, default=None) or self._html_search_regex(
63 r'(?s)<strong>Title:\s*</strong>(.+?)<', webpage, 'title')
64 description = self._html_search_meta('description', webpage)
65 thumbnail = self._og_search_thumbnail(webpage)
66
67 embed_params = [s.strip(" \r\n\t'") for s in self._search_regex(
68 r'(?s)\$\("#embedCode"\).html\(getEmbedCode\((.*?)\)',
69 webpage, 'embed params').split(',')]
70
71 (
72 _, speaker_id, story_id, story_duration,
73 speaker_type, great_life, _thumbnail, _has_subtitles,
74 story_filename, _story_order) = embed_params
75
76 is_great_life_series = great_life == 'true'
77 duration = int_or_none(story_duration)
78
79 # URL building, see: http://www.webofstories.com/scripts/player.js
80 ms_prefix = ''
81 if speaker_type.lower() == 'ms':
82 ms_prefix = 'mini_sites/'
83
84 if is_great_life_series:
85 mp4_url = '{0:}lives/{1:}/{2:}.mp4'.format(
86 self._VIDEO_DOMAIN, speaker_id, story_filename)
87 rtmp_ext = 'flv'
88 streamer = self._GREAT_LIFE_STREAMER
89 play_path = 'stories/{0:}/{1:}'.format(
90 speaker_id, story_filename)
91 else:
92 mp4_url = '{0:}{1:}{2:}/{3:}.mp4'.format(
93 self._VIDEO_DOMAIN, ms_prefix, speaker_id, story_filename)
94 rtmp_ext = 'mp4'
95 streamer = self._USER_STREAMER
96 play_path = 'mp4:{0:}{1:}/{2}.mp4'.format(
97 ms_prefix, speaker_id, story_filename)
98
99 formats = [{
100 'format_id': 'mp4_sd',
101 'url': mp4_url,
102 }, {
103 'format_id': 'rtmp_sd',
104 'page_url': url,
105 'url': streamer,
106 'ext': rtmp_ext,
107 'play_path': play_path,
108 }]
109
110 self._sort_formats(formats)
111
112 return {
113 'id': story_id,
114 'title': title,
115 'formats': formats,
116 'thumbnail': thumbnail,
117 'description': description,
118 'duration': duration,
119 }
120
121
122 class WebOfStoriesPlaylistIE(InfoExtractor):
123 _VALID_URL = r'https?://(?:www\.)?webofstories\.com/playAll/(?P<id>[^/]+)'
124 _TEST = {
125 'url': 'http://www.webofstories.com/playAll/donald.knuth',
126 'info_dict': {
127 'id': 'donald.knuth',
128 'title': 'Donald Knuth (Scientist)',
129 },
130 'playlist_mincount': 97,
131 }
132
133 def _real_extract(self, url):
134 playlist_id = self._match_id(url)
135
136 webpage = self._download_webpage(url, playlist_id)
137
138 entries = [
139 self.url_result(
140 'http://www.webofstories.com/play/%s' % video_id,
141 'WebOfStories', video_id=video_id)
142 for video_id in orderedSet(re.findall(r'\bid=["\']td_(\d+)', webpage))
143 ]
144
145 title = self._search_regex(
146 r'<div id="speakerName">\s*<span>([^<]+)</span>',
147 webpage, 'speaker', default=None)
148 if title:
149 field = self._search_regex(
150 r'<span id="primaryField">([^<]+)</span>',
151 webpage, 'field', default=None)
152 if field:
153 title += ' (%s)' % field
154
155 if not title:
156 title = self._search_regex(
157 r'<title>Play\s+all\s+stories\s*-\s*([^<]+)\s*-\s*Web\s+of\s+Stories</title>',
158 webpage, 'title')
159
160 return self.playlist_result(entries, playlist_id, title)