]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/pbs.py
debian/changelog: Annotate bugs to close.
[youtubedl] / youtube_dl / extractor / pbs.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from ..utils import (
7 ExtractorError,
8 determine_ext,
9 int_or_none,
10 unified_strdate,
11 US_RATINGS,
12 )
13
14
15 class PBSIE(InfoExtractor):
16 _VALID_URL = r'''(?x)https?://
17 (?:
18 # Direct video URL
19 video\.pbs\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
20 # Article with embedded player (or direct video)
21 (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
22 # Player
23 video\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
24 )
25 '''
26
27 _TESTS = [
28 {
29 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/',
30 'md5': 'ce1888486f0908d555a8093cac9a7362',
31 'info_dict': {
32 'id': '2365006249',
33 'ext': 'mp4',
34 'title': 'A More Perfect Union',
35 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
36 'duration': 3190,
37 },
38 },
39 {
40 'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
41 'md5': '143c98aa54a346738a3d78f54c925321',
42 'info_dict': {
43 'id': '2365297690',
44 'ext': 'mp4',
45 'title': 'Losing Iraq',
46 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
47 'duration': 5050,
48 },
49 },
50 {
51 'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/',
52 'md5': 'b19856d7f5351b17a5ab1dc6a64be633',
53 'info_dict': {
54 'id': '2201174722',
55 'ext': 'mp4',
56 'title': 'Cyber Schools Gain Popularity, but Quality Questions Persist',
57 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28',
58 'duration': 801,
59 },
60 },
61 {
62 'url': 'http://www.pbs.org/wnet/gperf/dudamel-conducts-verdi-requiem-hollywood-bowl-full-episode/3374/',
63 'md5': 'c62859342be2a0358d6c9eb306595978',
64 'info_dict': {
65 'id': '2365297708',
66 'ext': 'mp4',
67 'description': 'md5:68d87ef760660eb564455eb30ca464fe',
68 'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
69 'duration': 6559,
70 'thumbnail': 're:^https?://.*\.jpg$',
71 }
72 },
73 {
74 'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html',
75 'md5': '908f3e5473a693b266b84e25e1cf9703',
76 'info_dict': {
77 'id': '2365160389',
78 'display_id': 'killer-typhoon',
79 'ext': 'mp4',
80 'description': 'md5:c741d14e979fc53228c575894094f157',
81 'title': 'Killer Typhoon',
82 'duration': 3172,
83 'thumbnail': 're:^https?://.*\.jpg$',
84 'upload_date': '20140122',
85 }
86 },
87 {
88 'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
89 'info_dict': {
90 'id': 'united-states-of-secrets',
91 },
92 'playlist_count': 2,
93 }
94 ]
95
96 def _extract_webpage(self, url):
97 mobj = re.match(self._VALID_URL, url)
98
99 presumptive_id = mobj.group('presumptive_id')
100 display_id = presumptive_id
101 if presumptive_id:
102 webpage = self._download_webpage(url, display_id)
103
104 upload_date = unified_strdate(self._search_regex(
105 r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"',
106 webpage, 'upload date', default=None))
107
108 # tabbed frontline videos
109 tabbed_videos = re.findall(
110 r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"', webpage)
111 if tabbed_videos:
112 return tabbed_videos, presumptive_id, upload_date
113
114 MEDIA_ID_REGEXES = [
115 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", # frontline video embed
116 r'class="coveplayerid">([^<]+)<', # coveplayer
117 r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>', # jwplayer
118 ]
119
120 media_id = self._search_regex(
121 MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None)
122 if media_id:
123 return media_id, presumptive_id, upload_date
124
125 url = self._search_regex(
126 r'<iframe\s+(?:class|id)=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>',
127 webpage, 'player URL')
128 mobj = re.match(self._VALID_URL, url)
129
130 player_id = mobj.group('player_id')
131 if not display_id:
132 display_id = player_id
133 if player_id:
134 player_page = self._download_webpage(
135 url, display_id, note='Downloading player page',
136 errnote='Could not download player page')
137 video_id = self._search_regex(
138 r'<div\s+id="video_([0-9]+)"', player_page, 'video ID')
139 else:
140 video_id = mobj.group('id')
141 display_id = video_id
142
143 return video_id, display_id, None
144
145 def _real_extract(self, url):
146 video_id, display_id, upload_date = self._extract_webpage(url)
147
148 if isinstance(video_id, list):
149 entries = [self.url_result(
150 'http://video.pbs.org/video/%s' % vid_id, 'PBS', vid_id)
151 for vid_id in video_id]
152 return self.playlist_result(entries, display_id)
153
154 info = self._download_json(
155 'http://video.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
156 display_id)
157
158 formats = []
159 for encoding_name in ('recommended_encoding', 'alternate_encoding'):
160 redirect = info.get(encoding_name)
161 if not redirect:
162 continue
163 redirect_url = redirect.get('url')
164 if not redirect_url:
165 continue
166
167 redirect_info = self._download_json(
168 redirect_url + '?format=json', display_id,
169 'Downloading %s video url info' % encoding_name)
170
171 if redirect_info['status'] == 'error':
172 if redirect_info['http_code'] == 403:
173 message = (
174 'The video is not available in your region due to '
175 'right restrictions')
176 else:
177 message = redirect_info['message']
178 raise ExtractorError(message, expected=True)
179
180 format_url = redirect_info.get('url')
181 if not format_url:
182 continue
183
184 if determine_ext(format_url) == 'm3u8':
185 formats.extend(self._extract_m3u8_formats(
186 format_url, display_id, 'mp4', preference=1, m3u8_id='hls'))
187 else:
188 formats.append({
189 'url': format_url,
190 'format_id': redirect.get('eeid'),
191 })
192 self._sort_formats(formats)
193
194 rating_str = info.get('rating')
195 if rating_str is not None:
196 rating_str = rating_str.rpartition('-')[2]
197 age_limit = US_RATINGS.get(rating_str)
198
199 return {
200 'id': video_id,
201 'display_id': display_id,
202 'title': info['title'],
203 'description': info['program'].get('description'),
204 'thumbnail': info.get('image_url'),
205 'duration': int_or_none(info.get('duration')),
206 'age_limit': age_limit,
207 'upload_date': upload_date,
208 'formats': formats,
209 }