]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/youtube.py
Prepare for release.
[youtubedl] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 import json
4 import netrc
5 import re
6 import socket
7 import itertools
8
9 from .common import InfoExtractor, SearchInfoExtractor
10 from ..utils import (
11 compat_http_client,
12 compat_parse_qs,
13 compat_urllib_error,
14 compat_urllib_parse,
15 compat_urllib_request,
16 compat_str,
17
18 clean_html,
19 get_element_by_id,
20 ExtractorError,
21 unescapeHTML,
22 unified_strdate,
23 orderedSet,
24 )
25
26
27 class YoutubeIE(InfoExtractor):
28 IE_DESC = u'YouTube.com'
29 _VALID_URL = r"""^
30 (
31 (?:https?://)? # http(s):// (optional)
32 (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
33 tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains
34 (?:.*?\#/)? # handle anchor (#/) redirect urls
35 (?: # the various things that can precede the ID:
36 (?:(?:v|embed|e)/) # v/ or embed/ or e/
37 |(?: # or the v= param in all its forms
38 (?:watch|movie(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
39 (?:\?|\#!?) # the params delimiter ? or # or #!
40 (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
41 v=
42 )
43 )? # optional -> youtube.com/xxxx is OK
44 )? # all until now is optional -> you can pass the naked ID
45 ([0-9A-Za-z_-]+) # here is it! the YouTube video ID
46 (?(1).+)? # if we found the ID, everything can follow
47 $"""
48 _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
49 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
50 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
51 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
52 _NETRC_MACHINE = 'youtube'
53 # Listed in order of quality
54 _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
55 _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
56 _video_extensions = {
57 '13': '3gp',
58 '17': 'mp4',
59 '18': 'mp4',
60 '22': 'mp4',
61 '37': 'mp4',
62 '38': 'mp4',
63 '43': 'webm',
64 '44': 'webm',
65 '45': 'webm',
66 '46': 'webm',
67 }
68 _video_dimensions = {
69 '5': '240x400',
70 '6': '???',
71 '13': '???',
72 '17': '144x176',
73 '18': '360x640',
74 '22': '720x1280',
75 '34': '360x640',
76 '35': '480x854',
77 '37': '1080x1920',
78 '38': '3072x4096',
79 '43': '360x640',
80 '44': '480x854',
81 '45': '720x1280',
82 '46': '1080x1920',
83 }
84 IE_NAME = u'youtube'
85 _TESTS = [
86 {
87 u"url": u"http://www.youtube.com/watch?v=BaW_jenozKc",
88 u"file": u"BaW_jenozKc.mp4",
89 u"info_dict": {
90 u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
91 u"uploader": u"Philipp Hagemeister",
92 u"uploader_id": u"phihag",
93 u"upload_date": u"20121002",
94 u"description": u"test chars: \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
95 }
96 },
97 {
98 u"url": u"http://www.youtube.com/watch?v=1ltcDfZMA3U",
99 u"file": u"1ltcDfZMA3U.flv",
100 u"note": u"Test VEVO video (#897)",
101 u"info_dict": {
102 u"upload_date": u"20070518",
103 u"title": u"Maps - It Will Find You",
104 u"description": u"Music video by Maps performing It Will Find You.",
105 u"uploader": u"MuteUSA",
106 u"uploader_id": u"MuteUSA"
107 }
108 },
109 {
110 u"url": u"http://www.youtube.com/watch?v=UxxajLWwzqY",
111 u"file": u"UxxajLWwzqY.mp4",
112 u"note": u"Test generic use_cipher_signature video (#897)",
113 u"info_dict": {
114 u"upload_date": u"20120506",
115 u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
116 u"description": u"md5:b085c9804f5ab69f4adea963a2dceb3c",
117 u"uploader": u"IconaPop",
118 u"uploader_id": u"IconaPop"
119 }
120 },
121 {
122 u"url": u"https://www.youtube.com/watch?v=07FYdnEawAQ",
123 u"file": u"07FYdnEawAQ.mp4",
124 u"note": u"Test VEVO video with age protection (#956)",
125 u"info_dict": {
126 u"upload_date": u"20130703",
127 u"title": u"Justin Timberlake - Tunnel Vision (Explicit)",
128 u"description": u"md5:64249768eec3bc4276236606ea996373",
129 u"uploader": u"justintimberlakeVEVO",
130 u"uploader_id": u"justintimberlakeVEVO"
131 }
132 },
133 ]
134
135
136 @classmethod
137 def suitable(cls, url):
138 """Receives a URL and returns True if suitable for this IE."""
139 if YoutubePlaylistIE.suitable(url) or YoutubeSubscriptionsIE.suitable(url): return False
140 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
141
142 def report_lang(self):
143 """Report attempt to set language."""
144 self.to_screen(u'Setting language')
145
146 def report_video_webpage_download(self, video_id):
147 """Report attempt to download video webpage."""
148 self.to_screen(u'%s: Downloading video webpage' % video_id)
149
150 def report_video_info_webpage_download(self, video_id):
151 """Report attempt to download video info webpage."""
152 self.to_screen(u'%s: Downloading video info webpage' % video_id)
153
154 def report_video_subtitles_download(self, video_id):
155 """Report attempt to download video info webpage."""
156 self.to_screen(u'%s: Checking available subtitles' % video_id)
157
158 def report_video_subtitles_request(self, video_id, sub_lang, format):
159 """Report attempt to download video info webpage."""
160 self.to_screen(u'%s: Downloading video subtitles for %s.%s' % (video_id, sub_lang, format))
161
162 def report_video_subtitles_available(self, video_id, sub_lang_list):
163 """Report available subtitles."""
164 sub_lang = ",".join(list(sub_lang_list.keys()))
165 self.to_screen(u'%s: Available subtitles for video: %s' % (video_id, sub_lang))
166
167 def report_information_extraction(self, video_id):
168 """Report attempt to extract video information."""
169 self.to_screen(u'%s: Extracting video information' % video_id)
170
171 def report_unavailable_format(self, video_id, format):
172 """Report extracted video URL."""
173 self.to_screen(u'%s: Format %s not available' % (video_id, format))
174
175 def report_rtmp_download(self):
176 """Indicate the download will use the RTMP protocol."""
177 self.to_screen(u'RTMP download detected')
178
179 def _decrypt_signature(self, s):
180 """Turn the encrypted s field into a working signature"""
181
182 if len(s) == 88:
183 return s[48] + s[81:67:-1] + s[82] + s[66:62:-1] + s[85] + s[61:48:-1] + s[67] + s[47:12:-1] + s[3] + s[11:3:-1] + s[2] + s[12]
184 elif len(s) == 87:
185 return s[62] + s[82:62:-1] + s[83] + s[61:52:-1] + s[0] + s[51:2:-1]
186 elif len(s) == 86:
187 return s[2:63] + s[82] + s[64:82] + s[63]
188 elif len(s) == 85:
189 return s[76] + s[82:76:-1] + s[83] + s[75:60:-1] + s[0] + s[59:50:-1] + s[1] + s[49:2:-1]
190 elif len(s) == 84:
191 return s[83:36:-1] + s[2] + s[35:26:-1] + s[3] + s[25:3:-1] + s[26]
192 elif len(s) == 83:
193 return s[:81]
194 elif len(s) == 82:
195 return s[36] + s[79:67:-1] + s[81] + s[66:40:-1] + s[33] + s[39:36:-1] + s[40] + s[35] + s[0] + s[67] + s[32:0:-1] + s[34]
196
197 else:
198 raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
199
200 def _get_available_subtitles(self, video_id):
201 self.report_video_subtitles_download(video_id)
202 request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
203 try:
204 sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8')
205 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
206 return (u'unable to download video subtitles: %s' % compat_str(err), None)
207 sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
208 sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list)
209 if not sub_lang_list:
210 return (u'video doesn\'t have subtitles', None)
211 return sub_lang_list
212
213 def _list_available_subtitles(self, video_id):
214 sub_lang_list = self._get_available_subtitles(video_id)
215 self.report_video_subtitles_available(video_id, sub_lang_list)
216
217 def _request_subtitle(self, sub_lang, sub_name, video_id, format):
218 """
219 Return tuple:
220 (error_message, sub_lang, sub)
221 """
222 self.report_video_subtitles_request(video_id, sub_lang, format)
223 params = compat_urllib_parse.urlencode({
224 'lang': sub_lang,
225 'name': sub_name,
226 'v': video_id,
227 'fmt': format,
228 })
229 url = 'http://www.youtube.com/api/timedtext?' + params
230 try:
231 sub = compat_urllib_request.urlopen(url).read().decode('utf-8')
232 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
233 return (u'unable to download video subtitles: %s' % compat_str(err), None, None)
234 if not sub:
235 return (u'Did not fetch video subtitles', None, None)
236 return (None, sub_lang, sub)
237
238 def _request_automatic_caption(self, video_id, webpage):
239 """We need the webpage for getting the captions url, pass it as an
240 argument to speed up the process."""
241 sub_lang = self._downloader.params.get('subtitleslang') or 'en'
242 sub_format = self._downloader.params.get('subtitlesformat')
243 self.to_screen(u'%s: Looking for automatic captions' % video_id)
244 mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
245 err_msg = u'Couldn\'t find automatic captions for "%s"' % sub_lang
246 if mobj is None:
247 return [(err_msg, None, None)]
248 player_config = json.loads(mobj.group(1))
249 try:
250 args = player_config[u'args']
251 caption_url = args[u'ttsurl']
252 timestamp = args[u'timestamp']
253 params = compat_urllib_parse.urlencode({
254 'lang': 'en',
255 'tlang': sub_lang,
256 'fmt': sub_format,
257 'ts': timestamp,
258 'kind': 'asr',
259 })
260 subtitles_url = caption_url + '&' + params
261 sub = self._download_webpage(subtitles_url, video_id, u'Downloading automatic captions')
262 return [(None, sub_lang, sub)]
263 except KeyError:
264 return [(err_msg, None, None)]
265
266 def _extract_subtitle(self, video_id):
267 """
268 Return a list with a tuple:
269 [(error_message, sub_lang, sub)]
270 """
271 sub_lang_list = self._get_available_subtitles(video_id)
272 sub_format = self._downloader.params.get('subtitlesformat')
273 if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles
274 return [(sub_lang_list[0], None, None)]
275 if self._downloader.params.get('subtitleslang', False):
276 sub_lang = self._downloader.params.get('subtitleslang')
277 elif 'en' in sub_lang_list:
278 sub_lang = 'en'
279 else:
280 sub_lang = list(sub_lang_list.keys())[0]
281 if not sub_lang in sub_lang_list:
282 return [(u'no closed captions found in the specified language "%s"' % sub_lang, None, None)]
283
284 subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format)
285 return [subtitle]
286
287 def _extract_all_subtitles(self, video_id):
288 sub_lang_list = self._get_available_subtitles(video_id)
289 sub_format = self._downloader.params.get('subtitlesformat')
290 if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles
291 return [(sub_lang_list[0], None, None)]
292 subtitles = []
293 for sub_lang in sub_lang_list:
294 subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format)
295 subtitles.append(subtitle)
296 return subtitles
297
298 def _print_formats(self, formats):
299 print('Available formats:')
300 for x in formats:
301 print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')))
302
303 def _real_initialize(self):
304 if self._downloader is None:
305 return
306
307 # Set language
308 request = compat_urllib_request.Request(self._LANG_URL)
309 try:
310 self.report_lang()
311 compat_urllib_request.urlopen(request).read()
312 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
313 self._downloader.report_warning(u'unable to set language: %s' % compat_str(err))
314 return
315
316 (username, password) = self._get_login_info()
317
318 # No authentication to be performed
319 if username is None:
320 return
321
322 request = compat_urllib_request.Request(self._LOGIN_URL)
323 try:
324 login_page = compat_urllib_request.urlopen(request).read().decode('utf-8')
325 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
326 self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err))
327 return
328
329 galx = None
330 dsh = None
331 match = re.search(re.compile(r'<input.+?name="GALX".+?value="(.+?)"', re.DOTALL), login_page)
332 if match:
333 galx = match.group(1)
334
335 match = re.search(re.compile(r'<input.+?name="dsh".+?value="(.+?)"', re.DOTALL), login_page)
336 if match:
337 dsh = match.group(1)
338
339 # Log in
340 login_form_strs = {
341 u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
342 u'Email': username,
343 u'GALX': galx,
344 u'Passwd': password,
345 u'PersistentCookie': u'yes',
346 u'_utf8': u'霱',
347 u'bgresponse': u'js_disabled',
348 u'checkConnection': u'',
349 u'checkedDomains': u'youtube',
350 u'dnConn': u'',
351 u'dsh': dsh,
352 u'pstMsg': u'0',
353 u'rmShown': u'1',
354 u'secTok': u'',
355 u'signIn': u'Sign in',
356 u'timeStmp': u'',
357 u'service': u'youtube',
358 u'uilel': u'3',
359 u'hl': u'en_US',
360 }
361 # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
362 # chokes on unicode
363 login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
364 login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
365 request = compat_urllib_request.Request(self._LOGIN_URL, login_data)
366 try:
367 self.report_login()
368 login_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
369 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
370 self._downloader.report_warning(u'unable to log in: bad username or password')
371 return
372 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
373 self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
374 return
375
376 # Confirm age
377 age_form = {
378 'next_url': '/',
379 'action_confirm': 'Confirm',
380 }
381 request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
382 try:
383 self.report_age_confirmation()
384 compat_urllib_request.urlopen(request).read().decode('utf-8')
385 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
386 raise ExtractorError(u'Unable to confirm age: %s' % compat_str(err))
387
388 def _extract_id(self, url):
389 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
390 if mobj is None:
391 raise ExtractorError(u'Invalid URL: %s' % url)
392 video_id = mobj.group(2)
393 return video_id
394
395 def _real_extract(self, url):
396 if re.match(r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$', url):
397 self._downloader.report_warning(u'Did you forget to quote the URL? Remember that & is a meta-character in most shells, so you want to put the URL in quotes, like youtube-dl \'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\' (or simply youtube-dl BaW_jenozKc ).')
398
399 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
400 mobj = re.search(self._NEXT_URL_RE, url)
401 if mobj:
402 url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
403 video_id = self._extract_id(url)
404
405 # Get video webpage
406 self.report_video_webpage_download(video_id)
407 url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
408 request = compat_urllib_request.Request(url)
409 try:
410 video_webpage_bytes = compat_urllib_request.urlopen(request).read()
411 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
412 raise ExtractorError(u'Unable to download video webpage: %s' % compat_str(err))
413
414 video_webpage = video_webpage_bytes.decode('utf-8', 'ignore')
415
416 # Attempt to extract SWF player URL
417 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
418 if mobj is not None:
419 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
420 else:
421 player_url = None
422
423 # Get video info
424 self.report_video_info_webpage_download(video_id)
425 if re.search(r'player-age-gate-content">', video_webpage) is not None:
426 self.report_age_confirmation()
427 age_gate = True
428 # We simulate the access to the video from www.youtube.com/v/{video_id}
429 # this can be viewed without login into Youtube
430 data = compat_urllib_parse.urlencode({'video_id': video_id,
431 'el': 'embedded',
432 'gl': 'US',
433 'hl': 'en',
434 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
435 'asv': 3,
436 'sts':'1588',
437 })
438 video_info_url = 'https://www.youtube.com/get_video_info?' + data
439 video_info_webpage = self._download_webpage(video_info_url, video_id,
440 note=False,
441 errnote='unable to download video info webpage')
442 video_info = compat_parse_qs(video_info_webpage)
443 else:
444 age_gate = False
445 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
446 video_info_url = ('https://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
447 % (video_id, el_type))
448 video_info_webpage = self._download_webpage(video_info_url, video_id,
449 note=False,
450 errnote='unable to download video info webpage')
451 video_info = compat_parse_qs(video_info_webpage)
452 if 'token' in video_info:
453 break
454 if 'token' not in video_info:
455 if 'reason' in video_info:
456 raise ExtractorError(u'YouTube said: %s' % video_info['reason'][0], expected=True)
457 else:
458 raise ExtractorError(u'"token" parameter not in video info for unknown reason')
459
460 # Check for "rental" videos
461 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
462 raise ExtractorError(u'"rental" videos not supported')
463
464 # Start extracting information
465 self.report_information_extraction(video_id)
466
467 # uploader
468 if 'author' not in video_info:
469 raise ExtractorError(u'Unable to extract uploader name')
470 video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
471
472 # uploader_id
473 video_uploader_id = None
474 mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
475 if mobj is not None:
476 video_uploader_id = mobj.group(1)
477 else:
478 self._downloader.report_warning(u'unable to extract uploader nickname')
479
480 # title
481 if 'title' not in video_info:
482 raise ExtractorError(u'Unable to extract video title')
483 video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
484
485 # thumbnail image
486 # We try first to get a high quality image:
487 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
488 video_webpage, re.DOTALL)
489 if m_thumb is not None:
490 video_thumbnail = m_thumb.group(1)
491 elif 'thumbnail_url' not in video_info:
492 self._downloader.report_warning(u'unable to extract video thumbnail')
493 video_thumbnail = ''
494 else: # don't panic if we can't find it
495 video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
496
497 # upload date
498 upload_date = None
499 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
500 if mobj is not None:
501 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
502 upload_date = unified_strdate(upload_date)
503
504 # description
505 video_description = get_element_by_id("eow-description", video_webpage)
506 if video_description:
507 video_description = clean_html(video_description)
508 else:
509 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
510 if fd_mobj:
511 video_description = unescapeHTML(fd_mobj.group(1))
512 else:
513 video_description = u''
514
515 # subtitles
516 video_subtitles = None
517
518 if self._downloader.params.get('writesubtitles', False):
519 video_subtitles = self._extract_subtitle(video_id)
520 if video_subtitles:
521 (sub_error, sub_lang, sub) = video_subtitles[0]
522 if sub_error:
523 self._downloader.report_warning(sub_error)
524
525 if self._downloader.params.get('writeautomaticsub', False):
526 video_subtitles = self._request_automatic_caption(video_id, video_webpage)
527 (sub_error, sub_lang, sub) = video_subtitles[0]
528 if sub_error:
529 self._downloader.report_warning(sub_error)
530
531 if self._downloader.params.get('allsubtitles', False):
532 video_subtitles = self._extract_all_subtitles(video_id)
533 for video_subtitle in video_subtitles:
534 (sub_error, sub_lang, sub) = video_subtitle
535 if sub_error:
536 self._downloader.report_warning(sub_error)
537
538 if self._downloader.params.get('listsubtitles', False):
539 self._list_available_subtitles(video_id)
540 return
541
542 if 'length_seconds' not in video_info:
543 self._downloader.report_warning(u'unable to extract video duration')
544 video_duration = ''
545 else:
546 video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
547
548 # Decide which formats to download
549 req_format = self._downloader.params.get('format', None)
550
551 try:
552 mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
553 if not mobj:
554 raise ValueError('Could not find vevo ID')
555 info = json.loads(mobj.group(1))
556 args = info['args']
557 # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
558 # this signatures are encrypted
559 m_s = re.search(r'[&,]s=', args['url_encoded_fmt_stream_map'])
560 if m_s is not None:
561 self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
562 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
563 except ValueError:
564 pass
565
566 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
567 self.report_rtmp_download()
568 video_url_list = [(None, video_info['conn'][0])]
569 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
570 url_map = {}
571 for url_data_str in video_info['url_encoded_fmt_stream_map'][0].split(','):
572 url_data = compat_parse_qs(url_data_str)
573 if 'itag' in url_data and 'url' in url_data:
574 url = url_data['url'][0]
575 if 'sig' in url_data:
576 url += '&signature=' + url_data['sig'][0]
577 elif 's' in url_data:
578 if self._downloader.params.get('verbose'):
579 s = url_data['s'][0]
580 if age_gate:
581 player_version = self._search_regex(r'ad3-(.+?)\.swf',
582 video_info['ad3_module'][0], 'flash player',
583 fatal=False)
584 player = 'flash player %s' % player_version
585 else:
586 player = u'html5 player %s' % self._search_regex(r'html5player-(.+?)\.js', video_webpage,
587 'html5 player', fatal=False)
588 self.to_screen('encrypted signature length %d (%d.%d), itag %s, %s' %
589 (len(s), len(s.split('.')[0]), len(s.split('.')[1]), url_data['itag'][0], player))
590 signature = self._decrypt_signature(url_data['s'][0])
591 url += '&signature=' + signature
592 if 'ratebypass' not in url:
593 url += '&ratebypass=yes'
594 url_map[url_data['itag'][0]] = url
595
596 format_limit = self._downloader.params.get('format_limit', None)
597 available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
598 if format_limit is not None and format_limit in available_formats:
599 format_list = available_formats[available_formats.index(format_limit):]
600 else:
601 format_list = available_formats
602 existing_formats = [x for x in format_list if x in url_map]
603 if len(existing_formats) == 0:
604 raise ExtractorError(u'no known formats available for video')
605 if self._downloader.params.get('listformats', None):
606 self._print_formats(existing_formats)
607 return
608 if req_format is None or req_format == 'best':
609 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
610 elif req_format == 'worst':
611 video_url_list = [(existing_formats[-1], url_map[existing_formats[-1]])] # worst quality
612 elif req_format in ('-1', 'all'):
613 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
614 else:
615 # Specific formats. We pick the first in a slash-delimeted sequence.
616 # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
617 req_formats = req_format.split('/')
618 video_url_list = None
619 for rf in req_formats:
620 if rf in url_map:
621 video_url_list = [(rf, url_map[rf])]
622 break
623 if video_url_list is None:
624 raise ExtractorError(u'requested format not available')
625 else:
626 raise ExtractorError(u'no conn or url_encoded_fmt_stream_map information found in video info')
627
628 results = []
629 for format_param, video_real_url in video_url_list:
630 # Extension
631 video_extension = self._video_extensions.get(format_param, 'flv')
632
633 video_format = '{0} - {1}'.format(format_param if format_param else video_extension,
634 self._video_dimensions.get(format_param, '???'))
635
636 results.append({
637 'id': video_id,
638 'url': video_real_url,
639 'uploader': video_uploader,
640 'uploader_id': video_uploader_id,
641 'upload_date': upload_date,
642 'title': video_title,
643 'ext': video_extension,
644 'format': video_format,
645 'thumbnail': video_thumbnail,
646 'description': video_description,
647 'player_url': player_url,
648 'subtitles': video_subtitles,
649 'duration': video_duration
650 })
651 return results
652
653 class YoutubePlaylistIE(InfoExtractor):
654 IE_DESC = u'YouTube.com playlists'
655 _VALID_URL = r"""(?:
656 (?:https?://)?
657 (?:\w+\.)?
658 youtube\.com/
659 (?:
660 (?:course|view_play_list|my_playlists|artist|playlist|watch)
661 \? (?:.*?&)*? (?:p|a|list)=
662 | p/
663 )
664 ((?:PL|EC|UU)?[0-9A-Za-z-_]{10,})
665 .*
666 |
667 ((?:PL|EC|UU)[0-9A-Za-z-_]{10,})
668 )"""
669 _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/playlists/%s?max-results=%i&start-index=%i&v=2&alt=json&safeSearch=none'
670 _MAX_RESULTS = 50
671 IE_NAME = u'youtube:playlist'
672
673 @classmethod
674 def suitable(cls, url):
675 """Receives a URL and returns True if suitable for this IE."""
676 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
677
678 def _real_extract(self, url):
679 # Extract playlist id
680 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
681 if mobj is None:
682 raise ExtractorError(u'Invalid URL: %s' % url)
683
684 # Download playlist videos from API
685 playlist_id = mobj.group(1) or mobj.group(2)
686 page_num = 1
687 videos = []
688
689 while True:
690 url = self._TEMPLATE_URL % (playlist_id, self._MAX_RESULTS, self._MAX_RESULTS * (page_num - 1) + 1)
691 page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
692
693 try:
694 response = json.loads(page)
695 except ValueError as err:
696 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
697
698 if 'feed' not in response:
699 raise ExtractorError(u'Got a malformed response from YouTube API')
700 playlist_title = response['feed']['title']['$t']
701 if 'entry' not in response['feed']:
702 # Number of videos is a multiple of self._MAX_RESULTS
703 break
704
705 for entry in response['feed']['entry']:
706 index = entry['yt$position']['$t']
707 if 'media$group' in entry and 'media$player' in entry['media$group']:
708 videos.append((index, entry['media$group']['media$player']['url']))
709
710 if len(response['feed']['entry']) < self._MAX_RESULTS:
711 break
712 page_num += 1
713
714 videos = [v[1] for v in sorted(videos)]
715
716 url_results = [self.url_result(vurl, 'Youtube') for vurl in videos]
717 return [self.playlist_result(url_results, playlist_id, playlist_title)]
718
719
720 class YoutubeChannelIE(InfoExtractor):
721 IE_DESC = u'YouTube.com channels'
722 _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
723 _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
724 _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
725 _MORE_PAGES_URL = 'http://www.youtube.com/channel_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
726 IE_NAME = u'youtube:channel'
727
728 def extract_videos_from_page(self, page):
729 ids_in_page = []
730 for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
731 if mobj.group(1) not in ids_in_page:
732 ids_in_page.append(mobj.group(1))
733 return ids_in_page
734
735 def _real_extract(self, url):
736 # Extract channel id
737 mobj = re.match(self._VALID_URL, url)
738 if mobj is None:
739 raise ExtractorError(u'Invalid URL: %s' % url)
740
741 # Download channel page
742 channel_id = mobj.group(1)
743 video_ids = []
744 pagenum = 1
745
746 url = self._TEMPLATE_URL % (channel_id, pagenum)
747 page = self._download_webpage(url, channel_id,
748 u'Downloading page #%s' % pagenum)
749
750 # Extract video identifiers
751 ids_in_page = self.extract_videos_from_page(page)
752 video_ids.extend(ids_in_page)
753
754 # Download any subsequent channel pages using the json-based channel_ajax query
755 if self._MORE_PAGES_INDICATOR in page:
756 while True:
757 pagenum = pagenum + 1
758
759 url = self._MORE_PAGES_URL % (pagenum, channel_id)
760 page = self._download_webpage(url, channel_id,
761 u'Downloading page #%s' % pagenum)
762
763 page = json.loads(page)
764
765 ids_in_page = self.extract_videos_from_page(page['content_html'])
766 video_ids.extend(ids_in_page)
767
768 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
769 break
770
771 self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
772
773 urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids]
774 url_entries = [self.url_result(eurl, 'Youtube') for eurl in urls]
775 return [self.playlist_result(url_entries, channel_id)]
776
777
778 class YoutubeUserIE(InfoExtractor):
779 IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
780 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
781 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
782 _GDATA_PAGE_SIZE = 50
783 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
784 _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
785 IE_NAME = u'youtube:user'
786
787 def _real_extract(self, url):
788 # Extract username
789 mobj = re.match(self._VALID_URL, url)
790 if mobj is None:
791 raise ExtractorError(u'Invalid URL: %s' % url)
792
793 username = mobj.group(1)
794
795 # Download video ids using YouTube Data API. Result size per
796 # query is limited (currently to 50 videos) so we need to query
797 # page by page until there are no video ids - it means we got
798 # all of them.
799
800 video_ids = []
801 pagenum = 0
802
803 while True:
804 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
805
806 gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
807 page = self._download_webpage(gdata_url, username,
808 u'Downloading video ids from %d to %d' % (start_index, start_index + self._GDATA_PAGE_SIZE))
809
810 # Extract video identifiers
811 ids_in_page = []
812
813 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
814 if mobj.group(1) not in ids_in_page:
815 ids_in_page.append(mobj.group(1))
816
817 video_ids.extend(ids_in_page)
818
819 # A little optimization - if current page is not
820 # "full", ie. does not contain PAGE_SIZE video ids then
821 # we can assume that this page is the last one - there
822 # are no more ids on further pages - no need to query
823 # again.
824
825 if len(ids_in_page) < self._GDATA_PAGE_SIZE:
826 break
827
828 pagenum += 1
829
830 urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids]
831 url_results = [self.url_result(rurl, 'Youtube') for rurl in urls]
832 return [self.playlist_result(url_results, playlist_title = username)]
833
834 class YoutubeSearchIE(SearchInfoExtractor):
835 IE_DESC = u'YouTube.com searches'
836 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
837 _MAX_RESULTS = 1000
838 IE_NAME = u'youtube:search'
839 _SEARCH_KEY = 'ytsearch'
840
841 def report_download_page(self, query, pagenum):
842 """Report attempt to download search page with given number."""
843 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
844
845 def _get_n_results(self, query, n):
846 """Get a specified number of results for a query"""
847
848 video_ids = []
849 pagenum = 0
850 limit = n
851
852 while (50 * pagenum) < limit:
853 self.report_download_page(query, pagenum+1)
854 result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
855 request = compat_urllib_request.Request(result_url)
856 try:
857 data = compat_urllib_request.urlopen(request).read().decode('utf-8')
858 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
859 raise ExtractorError(u'Unable to download API page: %s' % compat_str(err))
860 api_response = json.loads(data)['data']
861
862 if not 'items' in api_response:
863 raise ExtractorError(u'[youtube] No video results')
864
865 new_ids = list(video['id'] for video in api_response['items'])
866 video_ids += new_ids
867
868 limit = min(n, api_response['totalItems'])
869 pagenum += 1
870
871 if len(video_ids) > n:
872 video_ids = video_ids[:n]
873 videos = [self.url_result('http://www.youtube.com/watch?v=%s' % id, 'Youtube') for id in video_ids]
874 return self.playlist_result(videos, query)
875
876
877 class YoutubeShowIE(InfoExtractor):
878 IE_DESC = u'YouTube.com (multi-season) shows'
879 _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
880 IE_NAME = u'youtube:show'
881
882 def _real_extract(self, url):
883 mobj = re.match(self._VALID_URL, url)
884 show_name = mobj.group(1)
885 webpage = self._download_webpage(url, show_name, u'Downloading show webpage')
886 # There's one playlist for each season of the show
887 m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
888 self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
889 return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
890
891
892 class YoutubeSubscriptionsIE(YoutubeIE):
893 """It's a subclass of YoutubeIE because we need to login"""
894 IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword(requires authentication)'
895 _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
896 IE_NAME = u'youtube:subscriptions'
897 _FEED_TEMPLATE = 'http://www.youtube.com/feed_ajax?action_load_system_feed=1&feed_name=subscriptions&paging=%s'
898 _PAGING_STEP = 30
899
900 # Overwrite YoutubeIE properties we don't want
901 _TESTS = []
902 @classmethod
903 def suitable(cls, url):
904 return re.match(cls._VALID_URL, url) is not None
905
906 def _real_initialize(self):
907 (username, password) = self._get_login_info()
908 if username is None:
909 raise ExtractorError(u'No login info available, needed for downloading the Youtube subscriptions.', expected=True)
910 super(YoutubeSubscriptionsIE, self)._real_initialize()
911
912 def _real_extract(self, url):
913 feed_entries = []
914 # The step argument is available only in 2.7 or higher
915 for i in itertools.count(0):
916 paging = i*self._PAGING_STEP
917 info = self._download_webpage(self._FEED_TEMPLATE % paging, 'feed',
918 u'Downloading page %s' % i)
919 info = json.loads(info)
920 feed_html = info['feed_html']
921 m_ids = re.finditer(r'"/watch\?v=(.*?)"', feed_html)
922 ids = orderedSet(m.group(1) for m in m_ids)
923 feed_entries.extend(self.url_result(id, 'Youtube') for id in ids)
924 if info['paging'] is None:
925 break
926 return self.playlist_result(feed_entries, playlist_title='Youtube Subscriptions')