]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/youtube.py
14a8bd6ea9ead38b15ae7fe41366248e6507882b
[youtubedl] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 import json
4 import netrc
5 import re
6 import socket
7
8 from .common import InfoExtractor, SearchInfoExtractor
9 from ..utils import (
10 compat_http_client,
11 compat_parse_qs,
12 compat_urllib_error,
13 compat_urllib_parse,
14 compat_urllib_request,
15 compat_str,
16
17 clean_html,
18 get_element_by_id,
19 ExtractorError,
20 unescapeHTML,
21 unified_strdate,
22 )
23
24
25 class YoutubeIE(InfoExtractor):
26 IE_DESC = u'YouTube.com'
27 _VALID_URL = r"""^
28 (
29 (?:https?://)? # http(s):// (optional)
30 (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
31 tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains
32 (?:.*?\#/)? # handle anchor (#/) redirect urls
33 (?: # the various things that can precede the ID:
34 (?:(?:v|embed|e)/) # v/ or embed/ or e/
35 |(?: # or the v= param in all its forms
36 (?:watch|movie(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
37 (?:\?|\#!?) # the params delimiter ? or # or #!
38 (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
39 v=
40 )
41 )? # optional -> youtube.com/xxxx is OK
42 )? # all until now is optional -> you can pass the naked ID
43 ([0-9A-Za-z_-]+) # here is it! the YouTube video ID
44 (?(1).+)? # if we found the ID, everything can follow
45 $"""
46 _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
47 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
48 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
49 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
50 _NETRC_MACHINE = 'youtube'
51 # Listed in order of quality
52 _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
53 _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
54 _video_extensions = {
55 '13': '3gp',
56 '17': 'mp4',
57 '18': 'mp4',
58 '22': 'mp4',
59 '37': 'mp4',
60 '38': 'mp4',
61 '43': 'webm',
62 '44': 'webm',
63 '45': 'webm',
64 '46': 'webm',
65 }
66 _video_dimensions = {
67 '5': '240x400',
68 '6': '???',
69 '13': '???',
70 '17': '144x176',
71 '18': '360x640',
72 '22': '720x1280',
73 '34': '360x640',
74 '35': '480x854',
75 '37': '1080x1920',
76 '38': '3072x4096',
77 '43': '360x640',
78 '44': '480x854',
79 '45': '720x1280',
80 '46': '1080x1920',
81 }
82 IE_NAME = u'youtube'
83 _TESTS = [
84 {
85 u"url": u"http://www.youtube.com/watch?v=BaW_jenozKc",
86 u"file": u"BaW_jenozKc.mp4",
87 u"info_dict": {
88 u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
89 u"uploader": u"Philipp Hagemeister",
90 u"uploader_id": u"phihag",
91 u"upload_date": u"20121002",
92 u"description": u"test chars: \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
93 }
94 },
95 {
96 u"url": u"http://www.youtube.com/watch?v=1ltcDfZMA3U",
97 u"file": u"1ltcDfZMA3U.flv",
98 u"note": u"Test VEVO video (#897)",
99 u"info_dict": {
100 u"upload_date": u"20070518",
101 u"title": u"Maps - It Will Find You",
102 u"description": u"Music video by Maps performing It Will Find You.",
103 u"uploader": u"MuteUSA",
104 u"uploader_id": u"MuteUSA"
105 }
106 },
107 {
108 u"url": u"http://www.youtube.com/watch?v=UxxajLWwzqY",
109 u"file": u"UxxajLWwzqY.mp4",
110 u"note": u"Test generic use_cipher_signature video (#897)",
111 u"info_dict": {
112 u"upload_date": u"20120506",
113 u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
114 u"description": u"md5:b085c9804f5ab69f4adea963a2dceb3c",
115 u"uploader": u"IconaPop",
116 u"uploader_id": u"IconaPop"
117 }
118 }
119 ]
120
121
122 @classmethod
123 def suitable(cls, url):
124 """Receives a URL and returns True if suitable for this IE."""
125 if YoutubePlaylistIE.suitable(url): return False
126 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
127
128 def report_lang(self):
129 """Report attempt to set language."""
130 self.to_screen(u'Setting language')
131
132 def report_login(self):
133 """Report attempt to log in."""
134 self.to_screen(u'Logging in')
135
136 def report_video_webpage_download(self, video_id):
137 """Report attempt to download video webpage."""
138 self.to_screen(u'%s: Downloading video webpage' % video_id)
139
140 def report_video_info_webpage_download(self, video_id):
141 """Report attempt to download video info webpage."""
142 self.to_screen(u'%s: Downloading video info webpage' % video_id)
143
144 def report_video_subtitles_download(self, video_id):
145 """Report attempt to download video info webpage."""
146 self.to_screen(u'%s: Checking available subtitles' % video_id)
147
148 def report_video_subtitles_request(self, video_id, sub_lang, format):
149 """Report attempt to download video info webpage."""
150 self.to_screen(u'%s: Downloading video subtitles for %s.%s' % (video_id, sub_lang, format))
151
152 def report_video_subtitles_available(self, video_id, sub_lang_list):
153 """Report available subtitles."""
154 sub_lang = ",".join(list(sub_lang_list.keys()))
155 self.to_screen(u'%s: Available subtitles for video: %s' % (video_id, sub_lang))
156
157 def report_information_extraction(self, video_id):
158 """Report attempt to extract video information."""
159 self.to_screen(u'%s: Extracting video information' % video_id)
160
161 def report_unavailable_format(self, video_id, format):
162 """Report extracted video URL."""
163 self.to_screen(u'%s: Format %s not available' % (video_id, format))
164
165 def report_rtmp_download(self):
166 """Indicate the download will use the RTMP protocol."""
167 self.to_screen(u'RTMP download detected')
168
169 def _decrypt_signature(self, s):
170 """Turn the encrypted s field into a working signature"""
171
172 if len(s) == 88:
173 return s[48] + s[81:67:-1] + s[82] + s[66:62:-1] + s[85] + s[61:48:-1] + s[67] + s[47:12:-1] + s[3] + s[11:3:-1] + s[2] + s[12]
174 elif len(s) == 87:
175 return s[62] + s[82:62:-1] + s[83] + s[61:52:-1] + s[0] + s[51:2:-1]
176 elif len(s) == 86:
177 return s[2:63] + s[82] + s[64:82] + s[63]
178 elif len(s) == 85:
179 return s[76] + s[82:76:-1] + s[83] + s[75:60:-1] + s[0] + s[59:50:-1] + s[1] + s[49:2:-1]
180 elif len(s) == 84:
181 return s[83:36:-1] + s[2] + s[35:26:-1] + s[3] + s[25:3:-1] + s[26]
182 elif len(s) == 83:
183 return s[52] + s[81:55:-1] + s[2] + s[54:52:-1] + s[82] + s[51:36:-1] + s[55] + s[35:2:-1] + s[36]
184 elif len(s) == 82:
185 return s[36] + s[79:67:-1] + s[81] + s[66:40:-1] + s[33] + s[39:36:-1] + s[40] + s[35] + s[0] + s[67] + s[32:0:-1] + s[34]
186
187 else:
188 raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
189
190 def _get_available_subtitles(self, video_id):
191 self.report_video_subtitles_download(video_id)
192 request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
193 try:
194 sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8')
195 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
196 return (u'unable to download video subtitles: %s' % compat_str(err), None)
197 sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
198 sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list)
199 if not sub_lang_list:
200 return (u'video doesn\'t have subtitles', None)
201 return sub_lang_list
202
203 def _list_available_subtitles(self, video_id):
204 sub_lang_list = self._get_available_subtitles(video_id)
205 self.report_video_subtitles_available(video_id, sub_lang_list)
206
207 def _request_subtitle(self, sub_lang, sub_name, video_id, format):
208 """
209 Return tuple:
210 (error_message, sub_lang, sub)
211 """
212 self.report_video_subtitles_request(video_id, sub_lang, format)
213 params = compat_urllib_parse.urlencode({
214 'lang': sub_lang,
215 'name': sub_name,
216 'v': video_id,
217 'fmt': format,
218 })
219 url = 'http://www.youtube.com/api/timedtext?' + params
220 try:
221 sub = compat_urllib_request.urlopen(url).read().decode('utf-8')
222 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
223 return (u'unable to download video subtitles: %s' % compat_str(err), None, None)
224 if not sub:
225 return (u'Did not fetch video subtitles', None, None)
226 return (None, sub_lang, sub)
227
228 def _request_automatic_caption(self, video_id, webpage):
229 """We need the webpage for getting the captions url, pass it as an
230 argument to speed up the process."""
231 sub_lang = self._downloader.params.get('subtitleslang') or 'en'
232 sub_format = self._downloader.params.get('subtitlesformat')
233 self.to_screen(u'%s: Looking for automatic captions' % video_id)
234 mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
235 err_msg = u'Couldn\'t find automatic captions for "%s"' % sub_lang
236 if mobj is None:
237 return [(err_msg, None, None)]
238 player_config = json.loads(mobj.group(1))
239 try:
240 args = player_config[u'args']
241 caption_url = args[u'ttsurl']
242 timestamp = args[u'timestamp']
243 params = compat_urllib_parse.urlencode({
244 'lang': 'en',
245 'tlang': sub_lang,
246 'fmt': sub_format,
247 'ts': timestamp,
248 'kind': 'asr',
249 })
250 subtitles_url = caption_url + '&' + params
251 sub = self._download_webpage(subtitles_url, video_id, u'Downloading automatic captions')
252 return [(None, sub_lang, sub)]
253 except KeyError:
254 return [(err_msg, None, None)]
255
256 def _extract_subtitle(self, video_id):
257 """
258 Return a list with a tuple:
259 [(error_message, sub_lang, sub)]
260 """
261 sub_lang_list = self._get_available_subtitles(video_id)
262 sub_format = self._downloader.params.get('subtitlesformat')
263 if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles
264 return [(sub_lang_list[0], None, None)]
265 if self._downloader.params.get('subtitleslang', False):
266 sub_lang = self._downloader.params.get('subtitleslang')
267 elif 'en' in sub_lang_list:
268 sub_lang = 'en'
269 else:
270 sub_lang = list(sub_lang_list.keys())[0]
271 if not sub_lang in sub_lang_list:
272 return [(u'no closed captions found in the specified language "%s"' % sub_lang, None, None)]
273
274 subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format)
275 return [subtitle]
276
277 def _extract_all_subtitles(self, video_id):
278 sub_lang_list = self._get_available_subtitles(video_id)
279 sub_format = self._downloader.params.get('subtitlesformat')
280 if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles
281 return [(sub_lang_list[0], None, None)]
282 subtitles = []
283 for sub_lang in sub_lang_list:
284 subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format)
285 subtitles.append(subtitle)
286 return subtitles
287
288 def _print_formats(self, formats):
289 print('Available formats:')
290 for x in formats:
291 print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')))
292
293 def _real_initialize(self):
294 if self._downloader is None:
295 return
296
297 username = None
298 password = None
299 downloader_params = self._downloader.params
300
301 # Attempt to use provided username and password or .netrc data
302 if downloader_params.get('username', None) is not None:
303 username = downloader_params['username']
304 password = downloader_params['password']
305 elif downloader_params.get('usenetrc', False):
306 try:
307 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
308 if info is not None:
309 username = info[0]
310 password = info[2]
311 else:
312 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
313 except (IOError, netrc.NetrcParseError) as err:
314 self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
315 return
316
317 # Set language
318 request = compat_urllib_request.Request(self._LANG_URL)
319 try:
320 self.report_lang()
321 compat_urllib_request.urlopen(request).read()
322 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
323 self._downloader.report_warning(u'unable to set language: %s' % compat_str(err))
324 return
325
326 # No authentication to be performed
327 if username is None:
328 return
329
330 request = compat_urllib_request.Request(self._LOGIN_URL)
331 try:
332 login_page = compat_urllib_request.urlopen(request).read().decode('utf-8')
333 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
334 self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err))
335 return
336
337 galx = None
338 dsh = None
339 match = re.search(re.compile(r'<input.+?name="GALX".+?value="(.+?)"', re.DOTALL), login_page)
340 if match:
341 galx = match.group(1)
342
343 match = re.search(re.compile(r'<input.+?name="dsh".+?value="(.+?)"', re.DOTALL), login_page)
344 if match:
345 dsh = match.group(1)
346
347 # Log in
348 login_form_strs = {
349 u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
350 u'Email': username,
351 u'GALX': galx,
352 u'Passwd': password,
353 u'PersistentCookie': u'yes',
354 u'_utf8': u'霱',
355 u'bgresponse': u'js_disabled',
356 u'checkConnection': u'',
357 u'checkedDomains': u'youtube',
358 u'dnConn': u'',
359 u'dsh': dsh,
360 u'pstMsg': u'0',
361 u'rmShown': u'1',
362 u'secTok': u'',
363 u'signIn': u'Sign in',
364 u'timeStmp': u'',
365 u'service': u'youtube',
366 u'uilel': u'3',
367 u'hl': u'en_US',
368 }
369 # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
370 # chokes on unicode
371 login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
372 login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
373 request = compat_urllib_request.Request(self._LOGIN_URL, login_data)
374 try:
375 self.report_login()
376 login_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
377 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
378 self._downloader.report_warning(u'unable to log in: bad username or password')
379 return
380 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
381 self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
382 return
383
384 # Confirm age
385 age_form = {
386 'next_url': '/',
387 'action_confirm': 'Confirm',
388 }
389 request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
390 try:
391 self.report_age_confirmation()
392 compat_urllib_request.urlopen(request).read().decode('utf-8')
393 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
394 raise ExtractorError(u'Unable to confirm age: %s' % compat_str(err))
395
396 def _extract_id(self, url):
397 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
398 if mobj is None:
399 raise ExtractorError(u'Invalid URL: %s' % url)
400 video_id = mobj.group(2)
401 return video_id
402
403 def _real_extract(self, url):
404 if re.match(r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$', url):
405 self._downloader.report_warning(u'Did you forget to quote the URL? Remember that & is a meta-character in most shells, so you want to put the URL in quotes, like youtube-dl \'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\' (or simply youtube-dl BaW_jenozKc ).')
406
407 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
408 mobj = re.search(self._NEXT_URL_RE, url)
409 if mobj:
410 url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
411 video_id = self._extract_id(url)
412
413 # Get video webpage
414 self.report_video_webpage_download(video_id)
415 url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
416 request = compat_urllib_request.Request(url)
417 try:
418 video_webpage_bytes = compat_urllib_request.urlopen(request).read()
419 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
420 raise ExtractorError(u'Unable to download video webpage: %s' % compat_str(err))
421
422 video_webpage = video_webpage_bytes.decode('utf-8', 'ignore')
423
424 # Attempt to extract SWF player URL
425 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
426 if mobj is not None:
427 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
428 else:
429 player_url = None
430
431 # Get video info
432 self.report_video_info_webpage_download(video_id)
433 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
434 video_info_url = ('https://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
435 % (video_id, el_type))
436 video_info_webpage = self._download_webpage(video_info_url, video_id,
437 note=False,
438 errnote='unable to download video info webpage')
439 video_info = compat_parse_qs(video_info_webpage)
440 if 'token' in video_info:
441 break
442 if 'token' not in video_info:
443 if 'reason' in video_info:
444 raise ExtractorError(u'YouTube said: %s' % video_info['reason'][0], expected=True)
445 else:
446 raise ExtractorError(u'"token" parameter not in video info for unknown reason')
447
448 # Check for "rental" videos
449 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
450 raise ExtractorError(u'"rental" videos not supported')
451
452 # Start extracting information
453 self.report_information_extraction(video_id)
454
455 # uploader
456 if 'author' not in video_info:
457 raise ExtractorError(u'Unable to extract uploader name')
458 video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
459
460 # uploader_id
461 video_uploader_id = None
462 mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
463 if mobj is not None:
464 video_uploader_id = mobj.group(1)
465 else:
466 self._downloader.report_warning(u'unable to extract uploader nickname')
467
468 # title
469 if 'title' not in video_info:
470 raise ExtractorError(u'Unable to extract video title')
471 video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
472
473 # thumbnail image
474 if 'thumbnail_url' not in video_info:
475 self._downloader.report_warning(u'unable to extract video thumbnail')
476 video_thumbnail = ''
477 else: # don't panic if we can't find it
478 video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
479
480 # upload date
481 upload_date = None
482 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
483 if mobj is not None:
484 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
485 upload_date = unified_strdate(upload_date)
486
487 # description
488 video_description = get_element_by_id("eow-description", video_webpage)
489 if video_description:
490 video_description = clean_html(video_description)
491 else:
492 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
493 if fd_mobj:
494 video_description = unescapeHTML(fd_mobj.group(1))
495 else:
496 video_description = u''
497
498 # subtitles
499 video_subtitles = None
500
501 if self._downloader.params.get('writesubtitles', False):
502 video_subtitles = self._extract_subtitle(video_id)
503 if video_subtitles:
504 (sub_error, sub_lang, sub) = video_subtitles[0]
505 if sub_error:
506 self._downloader.report_warning(sub_error)
507
508 if self._downloader.params.get('writeautomaticsub', False):
509 video_subtitles = self._request_automatic_caption(video_id, video_webpage)
510 (sub_error, sub_lang, sub) = video_subtitles[0]
511 if sub_error:
512 self._downloader.report_warning(sub_error)
513
514 if self._downloader.params.get('allsubtitles', False):
515 video_subtitles = self._extract_all_subtitles(video_id)
516 for video_subtitle in video_subtitles:
517 (sub_error, sub_lang, sub) = video_subtitle
518 if sub_error:
519 self._downloader.report_warning(sub_error)
520
521 if self._downloader.params.get('listsubtitles', False):
522 self._list_available_subtitles(video_id)
523 return
524
525 if 'length_seconds' not in video_info:
526 self._downloader.report_warning(u'unable to extract video duration')
527 video_duration = ''
528 else:
529 video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
530
531 # Decide which formats to download
532 req_format = self._downloader.params.get('format', None)
533
534 try:
535 mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
536 if not mobj:
537 raise ValueError('Could not find vevo ID')
538 info = json.loads(mobj.group(1))
539 args = info['args']
540 # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
541 # this signatures are encrypted
542 m_s = re.search(r'[&,]s=', args['url_encoded_fmt_stream_map'])
543 if m_s is not None:
544 self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
545 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
546 except ValueError:
547 pass
548
549 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
550 self.report_rtmp_download()
551 video_url_list = [(None, video_info['conn'][0])]
552 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
553 url_map = {}
554 for url_data_str in video_info['url_encoded_fmt_stream_map'][0].split(','):
555 url_data = compat_parse_qs(url_data_str)
556 if 'itag' in url_data and 'url' in url_data:
557 url = url_data['url'][0]
558 if 'sig' in url_data:
559 url += '&signature=' + url_data['sig'][0]
560 elif 's' in url_data:
561 if self._downloader.params.get('verbose'):
562 s = url_data['s'][0]
563 player = self._search_regex(r'html5player-(.+?)\.js', video_webpage,
564 'html5 player', fatal=False)
565 self.to_screen('encrypted signature length %d (%d.%d), itag %s, html5 player %s' %
566 (len(s), len(s.split('.')[0]), len(s.split('.')[1]), url_data['itag'][0], player))
567 signature = self._decrypt_signature(url_data['s'][0])
568 url += '&signature=' + signature
569 if 'ratebypass' not in url:
570 url += '&ratebypass=yes'
571 url_map[url_data['itag'][0]] = url
572
573 format_limit = self._downloader.params.get('format_limit', None)
574 available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
575 if format_limit is not None and format_limit in available_formats:
576 format_list = available_formats[available_formats.index(format_limit):]
577 else:
578 format_list = available_formats
579 existing_formats = [x for x in format_list if x in url_map]
580 if len(existing_formats) == 0:
581 raise ExtractorError(u'no known formats available for video')
582 if self._downloader.params.get('listformats', None):
583 self._print_formats(existing_formats)
584 return
585 if req_format is None or req_format == 'best':
586 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
587 elif req_format == 'worst':
588 video_url_list = [(existing_formats[-1], url_map[existing_formats[-1]])] # worst quality
589 elif req_format in ('-1', 'all'):
590 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
591 else:
592 # Specific formats. We pick the first in a slash-delimeted sequence.
593 # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
594 req_formats = req_format.split('/')
595 video_url_list = None
596 for rf in req_formats:
597 if rf in url_map:
598 video_url_list = [(rf, url_map[rf])]
599 break
600 if video_url_list is None:
601 raise ExtractorError(u'requested format not available')
602 else:
603 raise ExtractorError(u'no conn or url_encoded_fmt_stream_map information found in video info')
604
605 results = []
606 for format_param, video_real_url in video_url_list:
607 # Extension
608 video_extension = self._video_extensions.get(format_param, 'flv')
609
610 video_format = '{0} - {1}'.format(format_param if format_param else video_extension,
611 self._video_dimensions.get(format_param, '???'))
612
613 results.append({
614 'id': video_id,
615 'url': video_real_url,
616 'uploader': video_uploader,
617 'uploader_id': video_uploader_id,
618 'upload_date': upload_date,
619 'title': video_title,
620 'ext': video_extension,
621 'format': video_format,
622 'thumbnail': video_thumbnail,
623 'description': video_description,
624 'player_url': player_url,
625 'subtitles': video_subtitles,
626 'duration': video_duration
627 })
628 return results
629
630 class YoutubePlaylistIE(InfoExtractor):
631 IE_DESC = u'YouTube.com playlists'
632 _VALID_URL = r"""(?:
633 (?:https?://)?
634 (?:\w+\.)?
635 youtube\.com/
636 (?:
637 (?:course|view_play_list|my_playlists|artist|playlist|watch)
638 \? (?:.*?&)*? (?:p|a|list)=
639 | p/
640 )
641 ((?:PL|EC|UU)?[0-9A-Za-z-_]{10,})
642 .*
643 |
644 ((?:PL|EC|UU)[0-9A-Za-z-_]{10,})
645 )"""
646 _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/playlists/%s?max-results=%i&start-index=%i&v=2&alt=json&safeSearch=none'
647 _MAX_RESULTS = 50
648 IE_NAME = u'youtube:playlist'
649
650 @classmethod
651 def suitable(cls, url):
652 """Receives a URL and returns True if suitable for this IE."""
653 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
654
655 def _real_extract(self, url):
656 # Extract playlist id
657 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
658 if mobj is None:
659 raise ExtractorError(u'Invalid URL: %s' % url)
660
661 # Download playlist videos from API
662 playlist_id = mobj.group(1) or mobj.group(2)
663 page_num = 1
664 videos = []
665
666 while True:
667 url = self._TEMPLATE_URL % (playlist_id, self._MAX_RESULTS, self._MAX_RESULTS * (page_num - 1) + 1)
668 page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
669
670 try:
671 response = json.loads(page)
672 except ValueError as err:
673 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
674
675 if 'feed' not in response:
676 raise ExtractorError(u'Got a malformed response from YouTube API')
677 playlist_title = response['feed']['title']['$t']
678 if 'entry' not in response['feed']:
679 # Number of videos is a multiple of self._MAX_RESULTS
680 break
681
682 for entry in response['feed']['entry']:
683 index = entry['yt$position']['$t']
684 if 'media$group' in entry and 'media$player' in entry['media$group']:
685 videos.append((index, entry['media$group']['media$player']['url']))
686
687 if len(response['feed']['entry']) < self._MAX_RESULTS:
688 break
689 page_num += 1
690
691 videos = [v[1] for v in sorted(videos)]
692
693 url_results = [self.url_result(url, 'Youtube') for url in videos]
694 return [self.playlist_result(url_results, playlist_id, playlist_title)]
695
696
697 class YoutubeChannelIE(InfoExtractor):
698 IE_DESC = u'YouTube.com channels'
699 _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
700 _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
701 _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
702 _MORE_PAGES_URL = 'http://www.youtube.com/channel_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
703 IE_NAME = u'youtube:channel'
704
705 def extract_videos_from_page(self, page):
706 ids_in_page = []
707 for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
708 if mobj.group(1) not in ids_in_page:
709 ids_in_page.append(mobj.group(1))
710 return ids_in_page
711
712 def _real_extract(self, url):
713 # Extract channel id
714 mobj = re.match(self._VALID_URL, url)
715 if mobj is None:
716 raise ExtractorError(u'Invalid URL: %s' % url)
717
718 # Download channel page
719 channel_id = mobj.group(1)
720 video_ids = []
721 pagenum = 1
722
723 url = self._TEMPLATE_URL % (channel_id, pagenum)
724 page = self._download_webpage(url, channel_id,
725 u'Downloading page #%s' % pagenum)
726
727 # Extract video identifiers
728 ids_in_page = self.extract_videos_from_page(page)
729 video_ids.extend(ids_in_page)
730
731 # Download any subsequent channel pages using the json-based channel_ajax query
732 if self._MORE_PAGES_INDICATOR in page:
733 while True:
734 pagenum = pagenum + 1
735
736 url = self._MORE_PAGES_URL % (pagenum, channel_id)
737 page = self._download_webpage(url, channel_id,
738 u'Downloading page #%s' % pagenum)
739
740 page = json.loads(page)
741
742 ids_in_page = self.extract_videos_from_page(page['content_html'])
743 video_ids.extend(ids_in_page)
744
745 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
746 break
747
748 self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
749
750 urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids]
751 url_entries = [self.url_result(url, 'Youtube') for url in urls]
752 return [self.playlist_result(url_entries, channel_id)]
753
754
755 class YoutubeUserIE(InfoExtractor):
756 IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
757 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
758 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
759 _GDATA_PAGE_SIZE = 50
760 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
761 _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
762 IE_NAME = u'youtube:user'
763
764 def _real_extract(self, url):
765 # Extract username
766 mobj = re.match(self._VALID_URL, url)
767 if mobj is None:
768 raise ExtractorError(u'Invalid URL: %s' % url)
769
770 username = mobj.group(1)
771
772 # Download video ids using YouTube Data API. Result size per
773 # query is limited (currently to 50 videos) so we need to query
774 # page by page until there are no video ids - it means we got
775 # all of them.
776
777 video_ids = []
778 pagenum = 0
779
780 while True:
781 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
782
783 gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
784 page = self._download_webpage(gdata_url, username,
785 u'Downloading video ids from %d to %d' % (start_index, start_index + self._GDATA_PAGE_SIZE))
786
787 # Extract video identifiers
788 ids_in_page = []
789
790 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
791 if mobj.group(1) not in ids_in_page:
792 ids_in_page.append(mobj.group(1))
793
794 video_ids.extend(ids_in_page)
795
796 # A little optimization - if current page is not
797 # "full", ie. does not contain PAGE_SIZE video ids then
798 # we can assume that this page is the last one - there
799 # are no more ids on further pages - no need to query
800 # again.
801
802 if len(ids_in_page) < self._GDATA_PAGE_SIZE:
803 break
804
805 pagenum += 1
806
807 urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids]
808 url_results = [self.url_result(url, 'Youtube') for url in urls]
809 return [self.playlist_result(url_results, playlist_title = username)]
810
811 class YoutubeSearchIE(SearchInfoExtractor):
812 IE_DESC = u'YouTube.com searches'
813 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
814 _MAX_RESULTS = 1000
815 IE_NAME = u'youtube:search'
816 _SEARCH_KEY = 'ytsearch'
817
818 def report_download_page(self, query, pagenum):
819 """Report attempt to download search page with given number."""
820 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
821
822 def _get_n_results(self, query, n):
823 """Get a specified number of results for a query"""
824
825 video_ids = []
826 pagenum = 0
827 limit = n
828
829 while (50 * pagenum) < limit:
830 self.report_download_page(query, pagenum+1)
831 result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
832 request = compat_urllib_request.Request(result_url)
833 try:
834 data = compat_urllib_request.urlopen(request).read().decode('utf-8')
835 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
836 raise ExtractorError(u'Unable to download API page: %s' % compat_str(err))
837 api_response = json.loads(data)['data']
838
839 if not 'items' in api_response:
840 raise ExtractorError(u'[youtube] No video results')
841
842 new_ids = list(video['id'] for video in api_response['items'])
843 video_ids += new_ids
844
845 limit = min(n, api_response['totalItems'])
846 pagenum += 1
847
848 if len(video_ids) > n:
849 video_ids = video_ids[:n]
850 videos = [self.url_result('http://www.youtube.com/watch?v=%s' % id, 'Youtube') for id in video_ids]
851 return self.playlist_result(videos, query)
852
853
854 class YoutubeShowIE(InfoExtractor):
855 IE_DESC = u'YouTube.com (multi-season) shows'
856 _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
857 IE_NAME = u'youtube:show'
858
859 def _real_extract(self, url):
860 mobj = re.match(self._VALID_URL, url)
861 show_name = mobj.group(1)
862 webpage = self._download_webpage(url, show_name, u'Downloading show webpage')
863 # There's one playlist for each season of the show
864 m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
865 self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
866 return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]