]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/mtv.py
debian/copyright: Fix lintian's dep5-copyright-license-name-not-unique.
[youtubedl] / youtube_dl / extractor / mtv.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from ..compat import (
7 compat_urllib_parse,
8 compat_urllib_request,
9 compat_str,
10 )
11 from ..utils import (
12 ExtractorError,
13 find_xpath_attr,
14 fix_xml_ampersands,
15 HEADRequest,
16 unescapeHTML,
17 url_basename,
18 RegexNotFoundError,
19 )
20
21
22 def _media_xml_tag(tag):
23 return '{http://search.yahoo.com/mrss/}%s' % tag
24
25
26 class MTVServicesInfoExtractor(InfoExtractor):
27 _MOBILE_TEMPLATE = None
28 _LANG = None
29
30 @staticmethod
31 def _id_from_uri(uri):
32 return uri.split(':')[-1]
33
34 # This was originally implemented for ComedyCentral, but it also works here
35 @staticmethod
36 def _transform_rtmp_url(rtmp_video_url):
37 m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
38 if not m:
39 return rtmp_video_url
40 base = 'http://viacommtvstrmfs.fplive.net/'
41 return base + m.group('finalid')
42
43 def _get_feed_url(self, uri):
44 return self._FEED_URL
45
46 def _get_thumbnail_url(self, uri, itemdoc):
47 search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail'))
48 thumb_node = itemdoc.find(search_path)
49 if thumb_node is None:
50 return None
51 else:
52 return thumb_node.attrib['url']
53
54 def _extract_mobile_video_formats(self, mtvn_id):
55 webpage_url = self._MOBILE_TEMPLATE % mtvn_id
56 req = compat_urllib_request.Request(webpage_url)
57 # Otherwise we get a webpage that would execute some javascript
58 req.add_header('User-Agent', 'curl/7')
59 webpage = self._download_webpage(req, mtvn_id,
60 'Downloading mobile page')
61 metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
62 req = HEADRequest(metrics_url)
63 response = self._request_webpage(req, mtvn_id, 'Resolving url')
64 url = response.geturl()
65 # Transform the url to get the best quality:
66 url = re.sub(r'.+pxE=mp4', 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=0+_pxK=18639+_pxE=mp4', url, 1)
67 return [{'url': url, 'ext': 'mp4'}]
68
69 def _extract_video_formats(self, mdoc, mtvn_id):
70 if re.match(r'.*/(error_country_block\.swf|geoblock\.mp4)$', mdoc.find('.//src').text) is not None:
71 if mtvn_id is not None and self._MOBILE_TEMPLATE is not None:
72 self.to_screen('The normal version is not available from your '
73 'country, trying with the mobile version')
74 return self._extract_mobile_video_formats(mtvn_id)
75 raise ExtractorError('This video is not available from your country.',
76 expected=True)
77
78 formats = []
79 for rendition in mdoc.findall('.//rendition'):
80 try:
81 _, _, ext = rendition.attrib['type'].partition('/')
82 rtmp_video_url = rendition.find('./src').text
83 if rtmp_video_url.endswith('siteunavail.png'):
84 continue
85 formats.append({
86 'ext': ext,
87 'url': self._transform_rtmp_url(rtmp_video_url),
88 'format_id': rendition.get('bitrate'),
89 'width': int(rendition.get('width')),
90 'height': int(rendition.get('height')),
91 })
92 except (KeyError, TypeError):
93 raise ExtractorError('Invalid rendition field.')
94 self._sort_formats(formats)
95 return formats
96
97 def _extract_subtitles(self, mdoc, mtvn_id):
98 subtitles = {}
99 for transcript in mdoc.findall('.//transcript'):
100 if transcript.get('kind') != 'captions':
101 continue
102 lang = transcript.get('srclang')
103 subtitles[lang] = [{
104 'url': compat_str(typographic.get('src')),
105 'ext': typographic.get('format')
106 } for typographic in transcript.findall('./typographic')]
107 return subtitles
108
109 def _get_video_info(self, itemdoc):
110 uri = itemdoc.find('guid').text
111 video_id = self._id_from_uri(uri)
112 self.report_extraction(video_id)
113 mediagen_url = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content'))).attrib['url']
114 # Remove the templates, like &device={device}
115 mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', '', mediagen_url)
116 if 'acceptMethods' not in mediagen_url:
117 mediagen_url += '&acceptMethods=fms'
118
119 mediagen_doc = self._download_xml(mediagen_url, video_id,
120 'Downloading video urls')
121
122 item = mediagen_doc.find('./video/item')
123 if item is not None and item.get('type') == 'text':
124 message = '%s returned error: ' % self.IE_NAME
125 if item.get('code') is not None:
126 message += '%s - ' % item.get('code')
127 message += item.text
128 raise ExtractorError(message, expected=True)
129
130 description_node = itemdoc.find('description')
131 if description_node is not None:
132 description = description_node.text.strip()
133 else:
134 description = None
135
136 title_el = None
137 if title_el is None:
138 title_el = find_xpath_attr(
139 itemdoc, './/{http://search.yahoo.com/mrss/}category',
140 'scheme', 'urn:mtvn:video_title')
141 if title_el is None:
142 title_el = itemdoc.find('.//{http://search.yahoo.com/mrss/}title')
143 if title_el is None:
144 title_el = itemdoc.find('.//title')
145 if title_el.text is None:
146 title_el = None
147
148 title = title_el.text
149 if title is None:
150 raise ExtractorError('Could not find video title')
151 title = title.strip()
152
153 # This a short id that's used in the webpage urls
154 mtvn_id = None
155 mtvn_id_node = find_xpath_attr(itemdoc, './/{http://search.yahoo.com/mrss/}category',
156 'scheme', 'urn:mtvn:id')
157 if mtvn_id_node is not None:
158 mtvn_id = mtvn_id_node.text
159
160 return {
161 'title': title,
162 'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
163 'subtitles': self._extract_subtitles(mediagen_doc, mtvn_id),
164 'id': video_id,
165 'thumbnail': self._get_thumbnail_url(uri, itemdoc),
166 'description': description,
167 }
168
169 def _get_videos_info(self, uri):
170 video_id = self._id_from_uri(uri)
171 feed_url = self._get_feed_url(uri)
172 data = compat_urllib_parse.urlencode({'uri': uri})
173 info_url = feed_url + '?'
174 if self._LANG:
175 info_url += 'lang=%s&' % self._LANG
176 info_url += data
177 idoc = self._download_xml(
178 info_url, video_id,
179 'Downloading info', transform_source=fix_xml_ampersands)
180 return self.playlist_result(
181 [self._get_video_info(item) for item in idoc.findall('.//item')])
182
183 def _real_extract(self, url):
184 title = url_basename(url)
185 webpage = self._download_webpage(url, title)
186 try:
187 # the url can be http://media.mtvnservices.com/fb/{mgid}.swf
188 # or http://media.mtvnservices.com/{mgid}
189 og_url = self._og_search_video_url(webpage)
190 mgid = url_basename(og_url)
191 if mgid.endswith('.swf'):
192 mgid = mgid[:-4]
193 except RegexNotFoundError:
194 mgid = None
195
196 if mgid is None or ':' not in mgid:
197 mgid = self._search_regex(
198 [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
199 webpage, 'mgid')
200
201 videos_info = self._get_videos_info(mgid)
202 return videos_info
203
204
205 class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
206 IE_NAME = 'mtvservices:embedded'
207 _VALID_URL = r'https?://media\.mtvnservices\.com/embed/(?P<mgid>.+?)(\?|/|$)'
208
209 _TEST = {
210 # From http://www.thewrap.com/peter-dinklage-sums-up-game-of-thrones-in-45-seconds-video/
211 'url': 'http://media.mtvnservices.com/embed/mgid:uma:video:mtv.com:1043906/cp~vid%3D1043906%26uri%3Dmgid%3Auma%3Avideo%3Amtv.com%3A1043906',
212 'md5': 'cb349b21a7897164cede95bd7bf3fbb9',
213 'info_dict': {
214 'id': '1043906',
215 'ext': 'mp4',
216 'title': 'Peter Dinklage Sums Up \'Game Of Thrones\' In 45 Seconds',
217 'description': '"Sexy sexy sexy, stabby stabby stabby, beautiful language," says Peter Dinklage as he tries summarizing "Game of Thrones" in under a minute.',
218 },
219 }
220
221 def _get_feed_url(self, uri):
222 video_id = self._id_from_uri(uri)
223 site_id = uri.replace(video_id, '')
224 config_url = ('http://media.mtvnservices.com/pmt/e1/players/{0}/'
225 'context4/context5/config.xml'.format(site_id))
226 config_doc = self._download_xml(config_url, video_id)
227 feed_node = config_doc.find('.//feed')
228 feed_url = feed_node.text.strip().split('?')[0]
229 return feed_url
230
231 def _real_extract(self, url):
232 mobj = re.match(self._VALID_URL, url)
233 mgid = mobj.group('mgid')
234 return self._get_videos_info(mgid)
235
236
237 class MTVIE(MTVServicesInfoExtractor):
238 _VALID_URL = r'''(?x)^https?://
239 (?:(?:www\.)?mtv\.com/videos/.+?/(?P<videoid>[0-9]+)/[^/]+$|
240 m\.mtv\.com/videos/video\.rbml\?.*?id=(?P<mgid>[^&]+))'''
241
242 _FEED_URL = 'http://www.mtv.com/player/embed/AS3/rss/'
243
244 _TESTS = [
245 {
246 'url': 'http://www.mtv.com/videos/misc/853555/ours-vh1-storytellers.jhtml',
247 'md5': '850f3f143316b1e71fa56a4edfd6e0f8',
248 'info_dict': {
249 'id': '853555',
250 'ext': 'mp4',
251 'title': 'Taylor Swift - "Ours (VH1 Storytellers)"',
252 'description': 'Album: Taylor Swift performs "Ours" for VH1 Storytellers at Harvey Mudd College.',
253 },
254 },
255 ]
256
257 def _get_thumbnail_url(self, uri, itemdoc):
258 return 'http://mtv.mtvnimages.com/uri/' + uri
259
260 def _real_extract(self, url):
261 mobj = re.match(self._VALID_URL, url)
262 video_id = mobj.group('videoid')
263 uri = mobj.groupdict().get('mgid')
264 if uri is None:
265 webpage = self._download_webpage(url, video_id)
266
267 # Some videos come from Vevo.com
268 m_vevo = re.search(
269 r'(?s)isVevoVideo = true;.*?vevoVideoId = "(.*?)";', webpage)
270 if m_vevo:
271 vevo_id = m_vevo.group(1)
272 self.to_screen('Vevo video detected: %s' % vevo_id)
273 return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
274
275 uri = self._html_search_regex(r'/uri/(.*?)\?', webpage, 'uri')
276 return self._get_videos_info(uri)
277
278
279 class MTVIggyIE(MTVServicesInfoExtractor):
280 IE_NAME = 'mtviggy.com'
281 _VALID_URL = r'https?://www\.mtviggy\.com/videos/.+'
282 _TEST = {
283 'url': 'http://www.mtviggy.com/videos/arcade-fire-behind-the-scenes-at-the-biggest-music-experiment-yet/',
284 'info_dict': {
285 'id': '984696',
286 'ext': 'mp4',
287 'title': 'Arcade Fire: Behind the Scenes at the Biggest Music Experiment Yet',
288 }
289 }
290 _FEED_URL = 'http://all.mtvworldverticals.com/feed-xml/'