]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/pbs.py
debian/control: Add recommends on aria2 | wget | curl to use external downloaders.
[youtubedl] / youtube_dl / extractor / pbs.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from ..utils import (
7 ExtractorError,
8 unified_strdate,
9 US_RATINGS,
10 )
11
12
13 class PBSIE(InfoExtractor):
14 _VALID_URL = r'''(?x)https?://
15 (?:
16 # Direct video URL
17 video\.pbs\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
18 # Article with embedded player (or direct video)
19 (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
20 # Player
21 video\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
22 )
23 '''
24
25 _TESTS = [
26 {
27 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/',
28 'md5': 'ce1888486f0908d555a8093cac9a7362',
29 'info_dict': {
30 'id': '2365006249',
31 'ext': 'mp4',
32 'title': 'A More Perfect Union',
33 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a',
34 'duration': 3190,
35 },
36 },
37 {
38 'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/',
39 'md5': '143c98aa54a346738a3d78f54c925321',
40 'info_dict': {
41 'id': '2365297690',
42 'ext': 'mp4',
43 'title': 'Losing Iraq',
44 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
45 'duration': 5050,
46 },
47 },
48 {
49 'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/',
50 'md5': 'b19856d7f5351b17a5ab1dc6a64be633',
51 'info_dict': {
52 'id': '2201174722',
53 'ext': 'mp4',
54 'title': 'Cyber Schools Gain Popularity, but Quality Questions Persist',
55 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28',
56 'duration': 801,
57 },
58 },
59 {
60 'url': 'http://www.pbs.org/wnet/gperf/dudamel-conducts-verdi-requiem-hollywood-bowl-full-episode/3374/',
61 'md5': 'c62859342be2a0358d6c9eb306595978',
62 'info_dict': {
63 'id': '2365297708',
64 'ext': 'mp4',
65 'description': 'md5:68d87ef760660eb564455eb30ca464fe',
66 'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full',
67 'duration': 6559,
68 'thumbnail': 're:^https?://.*\.jpg$',
69 }
70 },
71 {
72 'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html',
73 'md5': '908f3e5473a693b266b84e25e1cf9703',
74 'info_dict': {
75 'id': '2365160389',
76 'display_id': 'killer-typhoon',
77 'ext': 'mp4',
78 'description': 'md5:c741d14e979fc53228c575894094f157',
79 'title': 'Killer Typhoon',
80 'duration': 3172,
81 'thumbnail': 're:^https?://.*\.jpg$',
82 'upload_date': '20140122',
83 }
84 },
85 {
86 'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
87 'info_dict': {
88 'id': 'united-states-of-secrets',
89 },
90 'playlist_count': 2,
91 }
92 ]
93
94 def _extract_webpage(self, url):
95 mobj = re.match(self._VALID_URL, url)
96
97 presumptive_id = mobj.group('presumptive_id')
98 display_id = presumptive_id
99 if presumptive_id:
100 webpage = self._download_webpage(url, display_id)
101
102 upload_date = unified_strdate(self._search_regex(
103 r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"',
104 webpage, 'upload date', default=None))
105
106 # tabbed frontline videos
107 tabbed_videos = re.findall(
108 r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"', webpage)
109 if tabbed_videos:
110 return tabbed_videos, presumptive_id, upload_date
111
112 MEDIA_ID_REGEXES = [
113 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", # frontline video embed
114 r'class="coveplayerid">([^<]+)<', # coveplayer
115 r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>', # jwplayer
116 ]
117
118 media_id = self._search_regex(
119 MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None)
120 if media_id:
121 return media_id, presumptive_id, upload_date
122
123 url = self._search_regex(
124 r'<iframe\s+(?:class|id)=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>',
125 webpage, 'player URL')
126 mobj = re.match(self._VALID_URL, url)
127
128 player_id = mobj.group('player_id')
129 if not display_id:
130 display_id = player_id
131 if player_id:
132 player_page = self._download_webpage(
133 url, display_id, note='Downloading player page',
134 errnote='Could not download player page')
135 video_id = self._search_regex(
136 r'<div\s+id="video_([0-9]+)"', player_page, 'video ID')
137 else:
138 video_id = mobj.group('id')
139 display_id = video_id
140
141 return video_id, display_id, None
142
143 def _real_extract(self, url):
144 video_id, display_id, upload_date = self._extract_webpage(url)
145
146 if isinstance(video_id, list):
147 entries = [self.url_result(
148 'http://video.pbs.org/video/%s' % vid_id, 'PBS', vid_id)
149 for vid_id in video_id]
150 return self.playlist_result(entries, display_id)
151
152 info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
153 info = self._download_json(info_url, display_id)
154
155 redirect_url = info['alternate_encoding']['url']
156 redirect_info = self._download_json(
157 redirect_url + '?format=json', display_id,
158 'Downloading video url info')
159 if redirect_info['status'] == 'error':
160 if redirect_info['http_code'] == 403:
161 message = (
162 'The video is not available in your region due to '
163 'right restrictions')
164 else:
165 message = redirect_info['message']
166 raise ExtractorError(message, expected=True)
167
168 rating_str = info.get('rating')
169 if rating_str is not None:
170 rating_str = rating_str.rpartition('-')[2]
171 age_limit = US_RATINGS.get(rating_str)
172
173 return {
174 'id': video_id,
175 'display_id': display_id,
176 'title': info['title'],
177 'url': redirect_info['url'],
178 'ext': 'mp4',
179 'description': info['program'].get('description'),
180 'thumbnail': info.get('image_url'),
181 'duration': info.get('duration'),
182 'age_limit': age_limit,
183 'upload_date': upload_date,
184 }