]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/youtube.py
Rapsys Git / youtubedl / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
debian/NEWS: Update with notes about split audio/video downloads.
[youtubedl] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 import collections
4 import errno
5 import io
6 import itertools
7 import json
8 import os.path
9 import re
10 import socket
11 import string
12 import struct
13 import traceback
14 import xml.etree.ElementTree
15 import zlib
16
17 from .common import InfoExtractor, SearchInfoExtractor
18 from .subtitles import SubtitlesInfoExtractor
19 from ..utils import (
20 compat_chr,
21 compat_http_client,
22 compat_parse_qs,
23 compat_urllib_error,
24 compat_urllib_parse,
25 compat_urllib_request,
26 compat_urlparse,
27 compat_str,
28
29 clean_html,
30 get_cachedir,
31 get_element_by_id,
32 ExtractorError,
33 unescapeHTML,
34 unified_strdate,
35 orderedSet,
36 write_json_file,
37 )
38
39 class YoutubeBaseInfoExtractor(InfoExtractor):
40 """Provide base functions for Youtube extractors"""
41 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
42 _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
43 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
44 _NETRC_MACHINE = 'youtube'
45 # If True it will raise an error if no login info is provided
46 _LOGIN_REQUIRED = False
47
48 def report_lang(self):
49 """Report attempt to set language."""
50 self.to_screen(u'Setting language')
51
52 def _set_language(self):
53 request = compat_urllib_request.Request(self._LANG_URL)
54 try:
55 self.report_lang()
56 compat_urllib_request.urlopen(request).read()
57 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
58 self._downloader.report_warning(u'unable to set language: %s' % compat_str(err))
59 return False
60 return True
61
62 def _login(self):
63 (username, password) = self._get_login_info()
64 # No authentication to be performed
65 if username is None:
66 if self._LOGIN_REQUIRED:
67 raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True)
68 return False
69
70 request = compat_urllib_request.Request(self._LOGIN_URL)
71 try:
72 login_page = compat_urllib_request.urlopen(request).read().decode('utf-8')
73 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
74 self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err))
75 return False
76
77 galx = None
78 dsh = None
79 match = re.search(re.compile(r'<input.+?name="GALX".+?value="(.+?)"', re.DOTALL), login_page)
80 if match:
81 galx = match.group(1)
82 match = re.search(re.compile(r'<input.+?name="dsh".+?value="(.+?)"', re.DOTALL), login_page)
83 if match:
84 dsh = match.group(1)
85
86 # Log in
87 login_form_strs = {
88 u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
89 u'Email': username,
90 u'GALX': galx,
91 u'Passwd': password,
92 u'PersistentCookie': u'yes',
93 u'_utf8': u'霱',
94 u'bgresponse': u'js_disabled',
95 u'checkConnection': u'',
96 u'checkedDomains': u'youtube',
97 u'dnConn': u'',
98 u'dsh': dsh,
99 u'pstMsg': u'0',
100 u'rmShown': u'1',
101 u'secTok': u'',
102 u'signIn': u'Sign in',
103 u'timeStmp': u'',
104 u'service': u'youtube',
105 u'uilel': u'3',
106 u'hl': u'en_US',
107 }
108 # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
109 # chokes on unicode
110 login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
111 login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
112 request = compat_urllib_request.Request(self._LOGIN_URL, login_data)
113 try:
114 self.report_login()
115 login_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
116 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
117 self._downloader.report_warning(u'unable to log in: bad username or password')
118 return False
119 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
120 self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
121 return False
122 return True
123
124 def _confirm_age(self):
125 age_form = {
126 'next_url': '/',
127 'action_confirm': 'Confirm',
128 }
129 request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
130 try:
131 self.report_age_confirmation()
132 compat_urllib_request.urlopen(request).read().decode('utf-8')
133 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
134 raise ExtractorError(u'Unable to confirm age: %s' % compat_str(err))
135 return True
136
137 def _real_initialize(self):
138 if self._downloader is None:
139 return
140 if not self._set_language():
141 return
142 if not self._login():
143 return
144 self._confirm_age()
145
146
147 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
148 IE_DESC = u'YouTube.com'
149 _VALID_URL = r"""^
150 (
151 (?:https?://)? # http(s):// (optional)
152 (?:(?:(?:(?:\w+\.)?youtube(?:-nocookie)?\.com/|
153 tube\.majestyc\.net/|
154 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
155 (?:.*?\#/)? # handle anchor (#/) redirect urls
156 (?: # the various things that can precede the ID:
157 (?:(?:v|embed|e)/) # v/ or embed/ or e/
158 |(?: # or the v= param in all its forms
159 (?:(?:watch|movie)(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
160 (?:\?|\#!?) # the params delimiter ? or # or #!
161 (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
162 v=
163 )
164 ))
165 |youtu\.be/ # just youtu.be/xxxx
166 )
167 )? # all until now is optional -> you can pass the naked ID
168 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
169 (?(1).+)? # if we found the ID, everything can follow
170 $"""
171 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
172 # Listed in order of quality
173 _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '36', '17', '13',
174 # Apple HTTP Live Streaming
175 '96', '95', '94', '93', '92', '132', '151',
176 # 3D
177 '85', '84', '102', '83', '101', '82', '100',
178 # Dash video
179 '138', '137', '248', '136', '247', '135', '246',
180 '245', '244', '134', '243', '133', '242', '160',
181 # Dash audio
182 '141', '172', '140', '171', '139',
183 ]
184 _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '36', '17', '13',
185 # Apple HTTP Live Streaming
186 '96', '95', '94', '93', '92', '132', '151',
187 # 3D
188 '85', '102', '84', '101', '83', '100', '82',
189 # Dash video
190 '138', '248', '137', '247', '136', '246', '245',
191 '244', '135', '243', '134', '242', '133', '160',
192 # Dash audio
193 '172', '141', '171', '140', '139',
194 ]
195 _video_formats_map = {
196 'flv': ['35', '34', '6', '5'],
197 '3gp': ['36', '17', '13'],
198 'mp4': ['38', '37', '22', '18'],
199 'webm': ['46', '45', '44', '43'],
200 }
201 _video_extensions = {
202 '13': '3gp',
203 '17': '3gp',
204 '18': 'mp4',
205 '22': 'mp4',
206 '36': '3gp',
207 '37': 'mp4',
208 '38': 'mp4',
209 '43': 'webm',
210 '44': 'webm',
211 '45': 'webm',
212 '46': 'webm',
213
214 # 3d videos
215 '82': 'mp4',
216 '83': 'mp4',
217 '84': 'mp4',
218 '85': 'mp4',
219 '100': 'webm',
220 '101': 'webm',
221 '102': 'webm',
222
223 # Apple HTTP Live Streaming
224 '92': 'mp4',
225 '93': 'mp4',
226 '94': 'mp4',
227 '95': 'mp4',
228 '96': 'mp4',
229 '132': 'mp4',
230 '151': 'mp4',
231
232 # Dash mp4
233 '133': 'mp4',
234 '134': 'mp4',
235 '135': 'mp4',
236 '136': 'mp4',
237 '137': 'mp4',
238 '138': 'mp4',
239 '139': 'mp4',
240 '140': 'mp4',
241 '141': 'mp4',
242 '160': 'mp4',
243
244 # Dash webm
245 '171': 'webm',
246 '172': 'webm',
247 '242': 'webm',
248 '243': 'webm',
249 '244': 'webm',
250 '245': 'webm',
251 '246': 'webm',
252 '247': 'webm',
253 '248': 'webm',
254 }
255 _video_dimensions = {
256 '5': '240x400',
257 '6': '???',
258 '13': '???',
259 '17': '144x176',
260 '18': '360x640',
261 '22': '720x1280',
262 '34': '360x640',
263 '35': '480x854',
264 '36': '240x320',
265 '37': '1080x1920',
266 '38': '3072x4096',
267 '43': '360x640',
268 '44': '480x854',
269 '45': '720x1280',
270 '46': '1080x1920',
271 '82': '360p',
272 '83': '480p',
273 '84': '720p',
274 '85': '1080p',
275 '92': '240p',
276 '93': '360p',
277 '94': '480p',
278 '95': '720p',
279 '96': '1080p',
280 '100': '360p',
281 '101': '480p',
282 '102': '720p',
283 '132': '240p',
284 '151': '72p',
285 '133': '240p',
286 '134': '360p',
287 '135': '480p',
288 '136': '720p',
289 '137': '1080p',
290 '138': '>1080p',
291 '139': '48k',
292 '140': '128k',
293 '141': '256k',
294 '160': '192p',
295 '171': '128k',
296 '172': '256k',
297 '242': '240p',
298 '243': '360p',
299 '244': '480p',
300 '245': '480p',
301 '246': '480p',
302 '247': '720p',
303 '248': '1080p',
304 }
305 _special_itags = {
306 '82': '3D',
307 '83': '3D',
308 '84': '3D',
309 '85': '3D',
310 '100': '3D',
311 '101': '3D',
312 '102': '3D',
313 '133': 'DASH Video',
314 '134': 'DASH Video',
315 '135': 'DASH Video',
316 '136': 'DASH Video',
317 '137': 'DASH Video',
318 '138': 'DASH Video',
319 '139': 'DASH Audio',
320 '140': 'DASH Audio',
321 '141': 'DASH Audio',
322 '160': 'DASH Video',
323 '171': 'DASH Audio',
324 '172': 'DASH Audio',
325 '242': 'DASH Video',
326 '243': 'DASH Video',
327 '244': 'DASH Video',
328 '245': 'DASH Video',
329 '246': 'DASH Video',
330 '247': 'DASH Video',
331 '248': 'DASH Video',
332 }
333
334 IE_NAME = u'youtube'
335 _TESTS = [
336 {
337 u"url": u"http://www.youtube.com/watch?v=BaW_jenozKc",
338 u"file": u"BaW_jenozKc.mp4",
339 u"info_dict": {
340 u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
341 u"uploader": u"Philipp Hagemeister",
342 u"uploader_id": u"phihag",
343 u"upload_date": u"20121002",
344 u"description": u"test chars: \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
345 }
346 },
347 {
348 u"url": u"http://www.youtube.com/watch?v=1ltcDfZMA3U",
349 u"file": u"1ltcDfZMA3U.flv",
350 u"note": u"Test VEVO video (#897)",
351 u"info_dict": {
352 u"upload_date": u"20070518",
353 u"title": u"Maps - It Will Find You",
354 u"description": u"Music video by Maps performing It Will Find You.",
355 u"uploader": u"MuteUSA",
356 u"uploader_id": u"MuteUSA"
357 }
358 },
359 {
360 u"url": u"http://www.youtube.com/watch?v=UxxajLWwzqY",
361 u"file": u"UxxajLWwzqY.mp4",
362 u"note": u"Test generic use_cipher_signature video (#897)",
363 u"info_dict": {
364 u"upload_date": u"20120506",
365 u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
366 u"description": u"md5:5b292926389560516e384ac437c0ec07",
367 u"uploader": u"Icona Pop",
368 u"uploader_id": u"IconaPop"
369 }
370 },
371 {
372 u"url": u"https://www.youtube.com/watch?v=07FYdnEawAQ",
373 u"file": u"07FYdnEawAQ.mp4",
374 u"note": u"Test VEVO video with age protection (#956)",
375 u"info_dict": {
376 u"upload_date": u"20130703",
377 u"title": u"Justin Timberlake - Tunnel Vision (Explicit)",
378 u"description": u"md5:64249768eec3bc4276236606ea996373",
379 u"uploader": u"justintimberlakeVEVO",
380 u"uploader_id": u"justintimberlakeVEVO"
381 }
382 },
383 ]
384
385
386 @classmethod
387 def suitable(cls, url):
388 """Receives a URL and returns True if suitable for this IE."""
389 if YoutubePlaylistIE.suitable(url): return False
390 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
391
392 def __init__(self, *args, **kwargs):
393 super(YoutubeIE, self).__init__(*args, **kwargs)
394 self._player_cache = {}
395
396 def report_video_webpage_download(self, video_id):
397 """Report attempt to download video webpage."""
398 self.to_screen(u'%s: Downloading video webpage' % video_id)
399
400 def report_video_info_webpage_download(self, video_id):
401 """Report attempt to download video info webpage."""
402 self.to_screen(u'%s: Downloading video info webpage' % video_id)
403
404 def report_information_extraction(self, video_id):
405 """Report attempt to extract video information."""
406 self.to_screen(u'%s: Extracting video information' % video_id)
407
408 def report_unavailable_format(self, video_id, format):
409 """Report extracted video URL."""
410 self.to_screen(u'%s: Format %s not available' % (video_id, format))
411
412 def report_rtmp_download(self):
413 """Indicate the download will use the RTMP protocol."""
414 self.to_screen(u'RTMP download detected')
415
416 def _extract_signature_function(self, video_id, player_url, slen):
417 id_m = re.match(r'.*-(?P<id>[a-zA-Z0-9_-]+)\.(?P<ext>[a-z]+)$',
418 player_url)
419 player_type = id_m.group('ext')
420 player_id = id_m.group('id')
421
422 # Read from filesystem cache
423 func_id = '%s_%s_%d' % (player_type, player_id, slen)
424 assert os.path.basename(func_id) == func_id
425 cache_dir = get_cachedir(self._downloader.params)
426
427 cache_enabled = cache_dir is not None
428 if cache_enabled:
429 cache_fn = os.path.join(os.path.expanduser(cache_dir),
430 u'youtube-sigfuncs',
431 func_id + '.json')
432 try:
433 with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
434 cache_spec = json.load(cachef)
435 return lambda s: u''.join(s[i] for i in cache_spec)
436 except IOError:
437 pass # No cache available
438
439 if player_type == 'js':
440 code = self._download_webpage(
441 player_url, video_id,
442 note=u'Downloading %s player %s' % (player_type, player_id),
443 errnote=u'Download of %s failed' % player_url)
444 res = self._parse_sig_js(code)
445 elif player_type == 'swf':
446 urlh = self._request_webpage(
447 player_url, video_id,
448 note=u'Downloading %s player %s' % (player_type, player_id),
449 errnote=u'Download of %s failed' % player_url)
450 code = urlh.read()
451 res = self._parse_sig_swf(code)
452 else:
453 assert False, 'Invalid player type %r' % player_type
454
455 if cache_enabled:
456 try:
457 test_string = u''.join(map(compat_chr, range(slen)))
458 cache_res = res(test_string)
459 cache_spec = [ord(c) for c in cache_res]
460 try:
461 os.makedirs(os.path.dirname(cache_fn))
462 except OSError as ose:
463 if ose.errno != errno.EEXIST:
464 raise
465 write_json_file(cache_spec, cache_fn)
466 except Exception:
467 tb = traceback.format_exc()
468 self._downloader.report_warning(
469 u'Writing cache to %r failed: %s' % (cache_fn, tb))
470
471 return res
472
473 def _print_sig_code(self, func, slen):
474 def gen_sig_code(idxs):
475 def _genslice(start, end, step):
476 starts = u'' if start == 0 else str(start)
477 ends = (u':%d' % (end+step)) if end + step >= 0 else u':'
478 steps = u'' if step == 1 else (u':%d' % step)
479 return u's[%s%s%s]' % (starts, ends, steps)
480
481 step = None
482 start = '(Never used)' # Quelch pyflakes warnings - start will be
483 # set as soon as step is set
484 for i, prev in zip(idxs[1:], idxs[:-1]):
485 if step is not None:
486 if i - prev == step:
487 continue
488 yield _genslice(start, prev, step)
489 step = None
490 continue
491 if i - prev in [-1, 1]:
492 step = i - prev
493 start = prev
494 continue
495 else:
496 yield u's[%d]' % prev
497 if step is None:
498 yield u's[%d]' % i
499 else:
500 yield _genslice(start, i, step)
501
502 test_string = u''.join(map(compat_chr, range(slen)))
503 cache_res = func(test_string)
504 cache_spec = [ord(c) for c in cache_res]
505 expr_code = u' + '.join(gen_sig_code(cache_spec))
506 code = u'if len(s) == %d:\n return %s\n' % (slen, expr_code)
507 self.to_screen(u'Extracted signature function:\n' + code)
508
509 def _parse_sig_js(self, jscode):
510 funcname = self._search_regex(
511 r'signature=([a-zA-Z]+)', jscode,
512 u'Initial JS player signature function name')
513
514 functions = {}
515
516 def argidx(varname):
517 return string.lowercase.index(varname)
518
519 def interpret_statement(stmt, local_vars, allow_recursion=20):
520 if allow_recursion < 0:
521 raise ExtractorError(u'Recursion limit reached')
522
523 if stmt.startswith(u'var '):
524 stmt = stmt[len(u'var '):]
525 ass_m = re.match(r'^(?P<out>[a-z]+)(?:\[(?P<index>[^\]]+)\])?' +
526 r'=(?P<expr>.*)$', stmt)
527 if ass_m:
528 if ass_m.groupdict().get('index'):
529 def assign(val):
530 lvar = local_vars[ass_m.group('out')]
531 idx = interpret_expression(ass_m.group('index'),
532 local_vars, allow_recursion)
533 assert isinstance(idx, int)
534 lvar[idx] = val
535 return val
536 expr = ass_m.group('expr')
537 else:
538 def assign(val):
539 local_vars[ass_m.group('out')] = val
540 return val
541 expr = ass_m.group('expr')
542 elif stmt.startswith(u'return '):
543 assign = lambda v: v
544 expr = stmt[len(u'return '):]
545 else:
546 raise ExtractorError(
547 u'Cannot determine left side of statement in %r' % stmt)
548
549 v = interpret_expression(expr, local_vars, allow_recursion)
550 return assign(v)
551
552 def interpret_expression(expr, local_vars, allow_recursion):
553 if expr.isdigit():
554 return int(expr)
555
556 if expr.isalpha():
557 return local_vars[expr]
558
559 m = re.match(r'^(?P<in>[a-z]+)\.(?P<member>.*)$', expr)
560 if m:
561 member = m.group('member')
562 val = local_vars[m.group('in')]
563 if member == 'split("")':
564 return list(val)
565 if member == 'join("")':
566 return u''.join(val)
567 if member == 'length':
568 return len(val)
569 if member == 'reverse()':
570 return val[::-1]
571 slice_m = re.match(r'slice\((?P<idx>.*)\)', member)
572 if slice_m:
573 idx = interpret_expression(
574 slice_m.group('idx'), local_vars, allow_recursion-1)
575 return val[idx:]
576
577 m = re.match(
578 r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
579 if m:
580 val = local_vars[m.group('in')]
581 idx = interpret_expression(m.group('idx'), local_vars,
582 allow_recursion-1)
583 return val[idx]
584
585 m = re.match(r'^(?P<a>.+?)(?P<op>[%])(?P<b>.+?)$', expr)
586 if m:
587 a = interpret_expression(m.group('a'),
588 local_vars, allow_recursion)
589 b = interpret_expression(m.group('b'),
590 local_vars, allow_recursion)
591 return a % b
592
593 m = re.match(
594 r'^(?P<func>[a-zA-Z]+)\((?P<args>[a-z0-9,]+)\)$', expr)
595 if m:
596 fname = m.group('func')
597 if fname not in functions:
598 functions[fname] = extract_function(fname)
599 argvals = [int(v) if v.isdigit() else local_vars[v]
600 for v in m.group('args').split(',')]
601 return functions[fname](argvals)
602 raise ExtractorError(u'Unsupported JS expression %r' % expr)
603
604 def extract_function(funcname):
605 func_m = re.search(
606 r'function ' + re.escape(funcname) +
607 r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
608 jscode)
609 argnames = func_m.group('args').split(',')
610
611 def resf(args):
612 local_vars = dict(zip(argnames, args))
613 for stmt in func_m.group('code').split(';'):
614 res = interpret_statement(stmt, local_vars)
615 return res
616 return resf
617
618 initial_function = extract_function(funcname)
619 return lambda s: initial_function([s])
620
621 def _parse_sig_swf(self, file_contents):
622 if file_contents[1:3] != b'WS':
623 raise ExtractorError(
624 u'Not an SWF file; header is %r' % file_contents[:3])
625 if file_contents[:1] == b'C':
626 content = zlib.decompress(file_contents[8:])
627 else:
628 raise NotImplementedError(u'Unsupported compression format %r' %
629 file_contents[:1])
630
631 def extract_tags(content):
632 pos = 0
633 while pos < len(content):
634 header16 = struct.unpack('<H', content[pos:pos+2])[0]
635 pos += 2
636 tag_code = header16 >> 6
637 tag_len = header16 & 0x3f
638 if tag_len == 0x3f:
639 tag_len = struct.unpack('<I', content[pos:pos+4])[0]
640 pos += 4
641 assert pos+tag_len <= len(content)
642 yield (tag_code, content[pos:pos+tag_len])
643 pos += tag_len
644
645 code_tag = next(tag
646 for tag_code, tag in extract_tags(content)
647 if tag_code == 82)
648 p = code_tag.index(b'\0', 4) + 1
649 code_reader = io.BytesIO(code_tag[p:])
650
651 # Parse ABC (AVM2 ByteCode)
652 def read_int(reader=None):
653 if reader is None:
654 reader = code_reader
655 res = 0
656 shift = 0
657 for _ in range(5):
658 buf = reader.read(1)
659 assert len(buf) == 1
660 b = struct.unpack('<B', buf)[0]
661 res = res | ((b & 0x7f) << shift)
662 if b & 0x80 == 0:
663 break
664 shift += 7
665 return res
666
667 def u30(reader=None):
668 res = read_int(reader)
669 assert res & 0xf0000000 == 0
670 return res
671 u32 = read_int
672
673 def s32(reader=None):
674 v = read_int(reader)
675 if v & 0x80000000 != 0:
676 v = - ((v ^ 0xffffffff) + 1)
677 return v
678
679 def read_string(reader=None):
680 if reader is None:
681 reader = code_reader
682 slen = u30(reader)
683 resb = reader.read(slen)
684 assert len(resb) == slen
685 return resb.decode('utf-8')
686
687 def read_bytes(count, reader=None):
688 if reader is None:
689 reader = code_reader
690 resb = reader.read(count)
691 assert len(resb) == count
692 return resb
693
694 def read_byte(reader=None):
695 resb = read_bytes(1, reader=reader)
696 res = struct.unpack('<B', resb)[0]
697 return res
698
699 # minor_version + major_version
700 read_bytes(2 + 2)
701
702 # Constant pool
703 int_count = u30()
704 for _c in range(1, int_count):
705 s32()
706 uint_count = u30()
707 for _c in range(1, uint_count):
708 u32()
709 double_count = u30()
710 read_bytes((double_count-1) * 8)
711 string_count = u30()
712 constant_strings = [u'']
713 for _c in range(1, string_count):
714 s = read_string()
715 constant_strings.append(s)
716 namespace_count = u30()
717 for _c in range(1, namespace_count):
718 read_bytes(1) # kind
719 u30() # name
720 ns_set_count = u30()
721 for _c in range(1, ns_set_count):
722 count = u30()
723 for _c2 in range(count):
724 u30()
725 multiname_count = u30()
726 MULTINAME_SIZES = {
727 0x07: 2, # QName
728 0x0d: 2, # QNameA
729 0x0f: 1, # RTQName
730 0x10: 1, # RTQNameA
731 0x11: 0, # RTQNameL
732 0x12: 0, # RTQNameLA
733 0x09: 2, # Multiname
734 0x0e: 2, # MultinameA
735 0x1b: 1, # MultinameL
736 0x1c: 1, # MultinameLA
737 }
738 multinames = [u'']
739 for _c in range(1, multiname_count):
740 kind = u30()
741 assert kind in MULTINAME_SIZES, u'Invalid multiname kind %r' % kind
742 if kind == 0x07:
743 u30() # namespace_idx
744 name_idx = u30()
745 multinames.append(constant_strings[name_idx])
746 else:
747 multinames.append('[MULTINAME kind: %d]' % kind)
748 for _c2 in range(MULTINAME_SIZES[kind]):
749 u30()
750
751 # Methods
752 method_count = u30()
753 MethodInfo = collections.namedtuple(
754 'MethodInfo',
755 ['NEED_ARGUMENTS', 'NEED_REST'])
756 method_infos = []
757 for method_id in range(method_count):
758 param_count = u30()
759 u30() # return type
760 for _ in range(param_count):
761 u30() # param type
762 u30() # name index (always 0 for youtube)
763 flags = read_byte()
764 if flags & 0x08 != 0:
765 # Options present
766 option_count = u30()
767 for c in range(option_count):
768 u30() # val
769 read_bytes(1) # kind
770 if flags & 0x80 != 0:
771 # Param names present
772 for _ in range(param_count):
773 u30() # param name
774 mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
775 method_infos.append(mi)
776
777 # Metadata
778 metadata_count = u30()
779 for _c in range(metadata_count):
780 u30() # name
781 item_count = u30()
782 for _c2 in range(item_count):
783 u30() # key
784 u30() # value
785
786 def parse_traits_info():
787 trait_name_idx = u30()
788 kind_full = read_byte()
789 kind = kind_full & 0x0f
790 attrs = kind_full >> 4
791 methods = {}
792 if kind in [0x00, 0x06]: # Slot or Const
793 u30() # Slot id
794 u30() # type_name_idx
795 vindex = u30()
796 if vindex != 0:
797 read_byte() # vkind
798 elif kind in [0x01, 0x02, 0x03]: # Method / Getter / Setter
799 u30() # disp_id
800 method_idx = u30()
801 methods[multinames[trait_name_idx]] = method_idx
802 elif kind == 0x04: # Class
803 u30() # slot_id
804 u30() # classi
805 elif kind == 0x05: # Function
806 u30() # slot_id
807 function_idx = u30()
808 methods[function_idx] = multinames[trait_name_idx]
809 else:
810 raise ExtractorError(u'Unsupported trait kind %d' % kind)
811
812 if attrs & 0x4 != 0: # Metadata present
813 metadata_count = u30()
814 for _c3 in range(metadata_count):
815 u30() # metadata index
816
817 return methods
818
819 # Classes
820 TARGET_CLASSNAME = u'SignatureDecipher'
821 searched_idx = multinames.index(TARGET_CLASSNAME)
822 searched_class_id = None
823 class_count = u30()
824 for class_id in range(class_count):
825 name_idx = u30()
826 if name_idx == searched_idx:
827 # We found the class we're looking for!
828 searched_class_id = class_id
829 u30() # super_name idx
830 flags = read_byte()
831 if flags & 0x08 != 0: # Protected namespace is present
832 u30() # protected_ns_idx
833 intrf_count = u30()
834 for _c2 in range(intrf_count):
835 u30()
836 u30() # iinit
837 trait_count = u30()
838 for _c2 in range(trait_count):
839 parse_traits_info()
840
841 if searched_class_id is None:
842 raise ExtractorError(u'Target class %r not found' %
843 TARGET_CLASSNAME)
844
845 method_names = {}
846 method_idxs = {}
847 for class_id in range(class_count):
848 u30() # cinit
849 trait_count = u30()
850 for _c2 in range(trait_count):
851 trait_methods = parse_traits_info()
852 if class_id == searched_class_id:
853 method_names.update(trait_methods.items())
854 method_idxs.update(dict(
855 (idx, name)
856 for name, idx in trait_methods.items()))
857
858 # Scripts
859 script_count = u30()
860 for _c in range(script_count):
861 u30() # init
862 trait_count = u30()
863 for _c2 in range(trait_count):
864 parse_traits_info()
865
866 # Method bodies
867 method_body_count = u30()
868 Method = collections.namedtuple('Method', ['code', 'local_count'])
869 methods = {}
870 for _c in range(method_body_count):
871 method_idx = u30()
872 u30() # max_stack
873 local_count = u30()
874 u30() # init_scope_depth
875 u30() # max_scope_depth
876 code_length = u30()
877 code = read_bytes(code_length)
878 if method_idx in method_idxs:
879 m = Method(code, local_count)
880 methods[method_idxs[method_idx]] = m
881 exception_count = u30()
882 for _c2 in range(exception_count):
883 u30() # from
884 u30() # to
885 u30() # target
886 u30() # exc_type
887 u30() # var_name
888 trait_count = u30()
889 for _c2 in range(trait_count):
890 parse_traits_info()
891
892 assert p + code_reader.tell() == len(code_tag)
893 assert len(methods) == len(method_idxs)
894
895 method_pyfunctions = {}
896
897 def extract_function(func_name):
898 if func_name in method_pyfunctions:
899 return method_pyfunctions[func_name]
900 if func_name not in methods:
901 raise ExtractorError(u'Cannot find function %r' % func_name)
902 m = methods[func_name]
903
904 def resfunc(args):
905 registers = ['(this)'] + list(args) + [None] * m.local_count
906 stack = []
907 coder = io.BytesIO(m.code)
908 while True:
909 opcode = struct.unpack('!B', coder.read(1))[0]
910 if opcode == 36: # pushbyte
911 v = struct.unpack('!B', coder.read(1))[0]
912 stack.append(v)
913 elif opcode == 44: # pushstring
914 idx = u30(coder)
915 stack.append(constant_strings[idx])
916 elif opcode == 48: # pushscope
917 # We don't implement the scope register, so we'll just
918 # ignore the popped value
919 stack.pop()
920 elif opcode == 70: # callproperty
921 index = u30(coder)
922 mname = multinames[index]
923 arg_count = u30(coder)
924 args = list(reversed(
925 [stack.pop() for _ in range(arg_count)]))
926 obj = stack.pop()
927 if mname == u'split':
928 assert len(args) == 1
929 assert isinstance(args[0], compat_str)
930 assert isinstance(obj, compat_str)
931 if args[0] == u'':
932 res = list(obj)
933 else:
934 res = obj.split(args[0])
935 stack.append(res)
936 elif mname == u'slice':
937 assert len(args) == 1
938 assert isinstance(args[0], int)
939 assert isinstance(obj, list)
940 res = obj[args[0]:]
941 stack.append(res)
942 elif mname == u'join':
943 assert len(args) == 1
944 assert isinstance(args[0], compat_str)
945 assert isinstance(obj, list)
946 res = args[0].join(obj)
947 stack.append(res)
948 elif mname in method_pyfunctions:
949 stack.append(method_pyfunctions[mname](args))
950 else:
951 raise NotImplementedError(
952 u'Unsupported property %r on %r'
953 % (mname, obj))
954 elif opcode == 72: # returnvalue
955 res = stack.pop()
956 return res
957 elif opcode == 79: # callpropvoid
958 index = u30(coder)
959 mname = multinames[index]
960 arg_count = u30(coder)
961 args = list(reversed(
962 [stack.pop() for _ in range(arg_count)]))
963 obj = stack.pop()
964 if mname == u'reverse':
965 assert isinstance(obj, list)
966 obj.reverse()
967 else:
968 raise NotImplementedError(
969 u'Unsupported (void) property %r on %r'
970 % (mname, obj))
971 elif opcode == 93: # findpropstrict
972 index = u30(coder)
973 mname = multinames[index]
974 res = extract_function(mname)
975 stack.append(res)
976 elif opcode == 97: # setproperty
977 index = u30(coder)
978 value = stack.pop()
979 idx = stack.pop()
980 obj = stack.pop()
981 assert isinstance(obj, list)
982 assert isinstance(idx, int)
983 obj[idx] = value
984 elif opcode == 98: # getlocal
985 index = u30(coder)
986 stack.append(registers[index])
987 elif opcode == 99: # setlocal
988 index = u30(coder)
989 value = stack.pop()
990 registers[index] = value
991 elif opcode == 102: # getproperty
992 index = u30(coder)
993 pname = multinames[index]
994 if pname == u'length':
995 obj = stack.pop()
996 assert isinstance(obj, list)
997 stack.append(len(obj))
998 else: # Assume attribute access
999 idx = stack.pop()
1000 assert isinstance(idx, int)
1001 obj = stack.pop()
1002 assert isinstance(obj, list)
1003 stack.append(obj[idx])
1004 elif opcode == 128: # coerce
1005 u30(coder)
1006 elif opcode == 133: # coerce_s
1007 assert isinstance(stack[-1], (type(None), compat_str))
1008 elif opcode == 164: # modulo
1009 value2 = stack.pop()
1010 value1 = stack.pop()
1011 res = value1 % value2
1012 stack.append(res)
1013 elif opcode == 208: # getlocal_0
1014 stack.append(registers[0])
1015 elif opcode == 209: # getlocal_1
1016 stack.append(registers[1])
1017 elif opcode == 210: # getlocal_2
1018 stack.append(registers[2])
1019 elif opcode == 211: # getlocal_3
1020 stack.append(registers[3])
1021 elif opcode == 214: # setlocal_2
1022 registers[2] = stack.pop()
1023 elif opcode == 215: # setlocal_3
1024 registers[3] = stack.pop()
1025 else:
1026 raise NotImplementedError(
1027 u'Unsupported opcode %d' % opcode)
1028
1029 method_pyfunctions[func_name] = resfunc
1030 return resfunc
1031
1032 initial_function = extract_function(u'decipher')
1033 return lambda s: initial_function([s])
1034
1035 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1036 """Turn the encrypted s field into a working signature"""
1037
1038 if player_url is not None:
1039 try:
1040 player_id = (player_url, len(s))
1041 if player_id not in self._player_cache:
1042 func = self._extract_signature_function(
1043 video_id, player_url, len(s)
1044 )
1045 self._player_cache[player_id] = func
1046 func = self._player_cache[player_id]
1047 if self._downloader.params.get('youtube_print_sig_code'):
1048 self._print_sig_code(func, len(s))
1049 return func(s)
1050 except Exception:
1051 tb = traceback.format_exc()
1052 self._downloader.report_warning(
1053 u'Automatic signature extraction failed: ' + tb)
1054
1055 self._downloader.report_warning(
1056 u'Warning: Falling back to static signature algorithm')
1057
1058 return self._static_decrypt_signature(
1059 s, video_id, player_url, age_gate)
1060
1061 def _static_decrypt_signature(self, s, video_id, player_url, age_gate):
1062 if age_gate:
1063 # The videos with age protection use another player, so the
1064 # algorithms can be different.
1065 if len(s) == 86:
1066 return s[2:63] + s[82] + s[64:82] + s[63]
1067
1068 if len(s) == 93:
1069 return s[86:29:-1] + s[88] + s[28:5:-1]
1070 elif len(s) == 92:
1071 return s[25] + s[3:25] + s[0] + s[26:42] + s[79] + s[43:79] + s[91] + s[80:83]
1072 elif len(s) == 91:
1073 return s[84:27:-1] + s[86] + s[26:5:-1]
1074 elif len(s) == 90:
1075 return s[25] + s[3:25] + s[2] + s[26:40] + s[77] + s[41:77] + s[89] + s[78:81]
1076 elif len(s) == 89:
1077 return s[84:78:-1] + s[87] + s[77:60:-1] + s[0] + s[59:3:-1]
1078 elif len(s) == 88:
1079 return s[7:28] + s[87] + s[29:45] + s[55] + s[46:55] + s[2] + s[56:87] + s[28]
1080 elif len(s) == 87:
1081 return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:]
1082 elif len(s) == 86:
1083 return s[80:72:-1] + s[16] + s[71:39:-1] + s[72] + s[38:16:-1] + s[82] + s[15::-1]
1084 elif len(s) == 85:
1085 return s[3:11] + s[0] + s[12:55] + s[84] + s[56:84]
1086 elif len(s) == 84:
1087 return s[78:70:-1] + s[14] + s[69:37:-1] + s[70] + s[36:14:-1] + s[80] + s[:14][::-1]
1088 elif len(s) == 83:
1089 return s[80:63:-1] + s[0] + s[62:0:-1] + s[63]
1090 elif len(s) == 82:
1091 return s[80:37:-1] + s[7] + s[36:7:-1] + s[0] + s[6:0:-1] + s[37]
1092 elif len(s) == 81:
1093 return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
1094 elif len(s) == 80:
1095 return s[1:19] + s[0] + s[20:68] + s[19] + s[69:80]
1096 elif len(s) == 79:
1097 return s[54] + s[77:54:-1] + s[39] + s[53:39:-1] + s[78] + s[38:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
1098
1099 else:
1100 raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
1101
1102 def _get_available_subtitles(self, video_id):
1103 try:
1104 sub_list = self._download_webpage(
1105 'http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1106 video_id, note=False)
1107 except ExtractorError as err:
1108 self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
1109 return {}
1110 lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
1111
1112 sub_lang_list = {}
1113 for l in lang_list:
1114 lang = l[1]
1115 params = compat_urllib_parse.urlencode({
1116 'lang': lang,
1117 'v': video_id,
1118 'fmt': self._downloader.params.get('subtitlesformat'),
1119 'name': l[0],
1120 })
1121 url = u'http://www.youtube.com/api/timedtext?' + params
1122 sub_lang_list[lang] = url
1123 if not sub_lang_list:
1124 self._downloader.report_warning(u'video doesn\'t have subtitles')
1125 return {}
1126 return sub_lang_list
1127
1128 def _get_available_automatic_caption(self, video_id, webpage):
1129 """We need the webpage for getting the captions url, pass it as an
1130 argument to speed up the process."""
1131 sub_format = self._downloader.params.get('subtitlesformat')
1132 self.to_screen(u'%s: Looking for automatic captions' % video_id)
1133 mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
1134 err_msg = u'Couldn\'t find automatic captions for %s' % video_id
1135 if mobj is None:
1136 self._downloader.report_warning(err_msg)
1137 return {}
1138 player_config = json.loads(mobj.group(1))
1139 try:
1140 args = player_config[u'args']
1141 caption_url = args[u'ttsurl']
1142 timestamp = args[u'timestamp']
1143 # We get the available subtitles
1144 list_params = compat_urllib_parse.urlencode({
1145 'type': 'list',
1146 'tlangs': 1,
1147 'asrs': 1,
1148 })
1149 list_url = caption_url + '&' + list_params
1150 list_page = self._download_webpage(list_url, video_id)
1151 caption_list = xml.etree.ElementTree.fromstring(list_page.encode('utf-8'))
1152 original_lang_node = caption_list.find('track')
1153 if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' :
1154 self._downloader.report_warning(u'Video doesn\'t have automatic captions')
1155 return {}
1156 original_lang = original_lang_node.attrib['lang_code']
1157
1158 sub_lang_list = {}
1159 for lang_node in caption_list.findall('target'):
1160 sub_lang = lang_node.attrib['lang_code']
1161 params = compat_urllib_parse.urlencode({
1162 'lang': original_lang,
1163 'tlang': sub_lang,
1164 'fmt': sub_format,
1165 'ts': timestamp,
1166 'kind': 'asr',
1167 })
1168 sub_lang_list[sub_lang] = caption_url + '&' + params
1169 return sub_lang_list
1170 # An extractor error can be raise by the download process if there are
1171 # no automatic captions but there are subtitles
1172 except (KeyError, ExtractorError):
1173 self._downloader.report_warning(err_msg)
1174 return {}
1175
1176 def _print_formats(self, formats):
1177 print('Available formats:')
1178 for x in formats:
1179 print('%s\t:\t%s\t[%s]%s' %(x, self._video_extensions.get(x, 'flv'),
1180 self._video_dimensions.get(x, '???'),
1181 ' ('+self._special_itags[x]+')' if x in self._special_itags else ''))
1182
1183 def _extract_id(self, url):
1184 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1185 if mobj is None:
1186 raise ExtractorError(u'Invalid URL: %s' % url)
1187 video_id = mobj.group(2)
1188 return video_id
1189
1190 def _get_video_url_list(self, url_map):
1191 """
1192 Transform a dictionary in the format {itag:url} to a list of (itag, url)
1193 with the requested formats.
1194 """
1195 req_format = self._downloader.params.get('format', None)
1196 format_limit = self._downloader.params.get('format_limit', None)
1197 available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
1198 if format_limit is not None and format_limit in available_formats:
1199 format_list = available_formats[available_formats.index(format_limit):]
1200 else:
1201 format_list = available_formats
1202 existing_formats = [x for x in format_list if x in url_map]
1203 if len(existing_formats) == 0:
1204 raise ExtractorError(u'no known formats available for video')
1205 if self._downloader.params.get('listformats', None):
1206 self._print_formats(existing_formats)
1207 return
1208 if req_format is None or req_format == 'best':
1209 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
1210 elif req_format == 'worst':
1211 video_url_list = [(existing_formats[-1], url_map[existing_formats[-1]])] # worst quality
1212 elif req_format in ('-1', 'all'):
1213 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
1214 else:
1215 # Specific formats. We pick the first in a slash-delimeted sequence.
1216 # Format can be specified as itag or 'mp4' or 'flv' etc. We pick the highest quality
1217 # available in the specified format. For example,
1218 # if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
1219 # if '1/mp4/3/4' is requested and '1' and '5' (is a mp4) are available, we pick '1'.
1220 # if '1/mp4/3/4' is requested and '4' and '5' (is a mp4) are available, we pick '5'.
1221 req_formats = req_format.split('/')
1222 video_url_list = None
1223 for rf in req_formats:
1224 if rf in url_map:
1225 video_url_list = [(rf, url_map[rf])]
1226 break
1227 if rf in self._video_formats_map:
1228 for srf in self._video_formats_map[rf]:
1229 if srf in url_map:
1230 video_url_list = [(srf, url_map[srf])]
1231 break
1232 else:
1233 continue
1234 break
1235 if video_url_list is None:
1236 raise ExtractorError(u'requested format not available')
1237 return video_url_list
1238
1239 def _extract_from_m3u8(self, manifest_url, video_id):
1240 url_map = {}
1241 def _get_urls(_manifest):
1242 lines = _manifest.split('\n')
1243 urls = filter(lambda l: l and not l.startswith('#'),
1244 lines)
1245 return urls
1246 manifest = self._download_webpage(manifest_url, video_id, u'Downloading formats manifest')
1247 formats_urls = _get_urls(manifest)
1248 for format_url in formats_urls:
1249 itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
1250 url_map[itag] = format_url
1251 return url_map
1252
1253 def _extract_annotations(self, video_id):
1254 url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1255 return self._download_webpage(url, video_id, note=u'Searching for annotations.', errnote=u'Unable to download video annotations.')
1256
1257 def _real_extract(self, url):
1258 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1259 mobj = re.search(self._NEXT_URL_RE, url)
1260 if mobj:
1261 url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
1262 video_id = self._extract_id(url)
1263
1264 # Get video webpage
1265 self.report_video_webpage_download(video_id)
1266 url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
1267 request = compat_urllib_request.Request(url)
1268 try:
1269 video_webpage_bytes = compat_urllib_request.urlopen(request).read()
1270 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1271 raise ExtractorError(u'Unable to download video webpage: %s' % compat_str(err))
1272
1273 video_webpage = video_webpage_bytes.decode('utf-8', 'ignore')
1274
1275 # Attempt to extract SWF player URL
1276 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1277 if mobj is not None:
1278 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1279 else:
1280 player_url = None
1281
1282 # Get video info
1283 self.report_video_info_webpage_download(video_id)
1284 if re.search(r'player-age-gate-content">', video_webpage) is not None:
1285 self.report_age_confirmation()
1286 age_gate = True
1287 # We simulate the access to the video from www.youtube.com/v/{video_id}
1288 # this can be viewed without login into Youtube
1289 data = compat_urllib_parse.urlencode({'video_id': video_id,
1290 'el': 'embedded',
1291 'gl': 'US',
1292 'hl': 'en',
1293 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1294 'asv': 3,
1295 'sts':'1588',
1296 })
1297 video_info_url = 'https://www.youtube.com/get_video_info?' + data
1298 video_info_webpage = self._download_webpage(video_info_url, video_id,
1299 note=False,
1300 errnote='unable to download video info webpage')
1301 video_info = compat_parse_qs(video_info_webpage)
1302 else:
1303 age_gate = False
1304 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1305 video_info_url = ('https://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1306 % (video_id, el_type))
1307 video_info_webpage = self._download_webpage(video_info_url, video_id,
1308 note=False,
1309 errnote='unable to download video info webpage')
1310 video_info = compat_parse_qs(video_info_webpage)
1311 if 'token' in video_info:
1312 break
1313 if 'token' not in video_info:
1314 if 'reason' in video_info:
1315 raise ExtractorError(u'YouTube said: %s' % video_info['reason'][0], expected=True)
1316 else:
1317 raise ExtractorError(u'"token" parameter not in video info for unknown reason')
1318
1319 # Check for "rental" videos
1320 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1321 raise ExtractorError(u'"rental" videos not supported')
1322
1323 # Start extracting information
1324 self.report_information_extraction(video_id)
1325
1326 # uploader
1327 if 'author' not in video_info:
1328 raise ExtractorError(u'Unable to extract uploader name')
1329 video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
1330
1331 # uploader_id
1332 video_uploader_id = None
1333 mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
1334 if mobj is not None:
1335 video_uploader_id = mobj.group(1)
1336 else:
1337 self._downloader.report_warning(u'unable to extract uploader nickname')
1338
1339 # title
1340 if 'title' in video_info:
1341 video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
1342 else:
1343 self._downloader.report_warning(u'Unable to extract video title')
1344 video_title = u'_'
1345
1346 # thumbnail image
1347 # We try first to get a high quality image:
1348 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1349 video_webpage, re.DOTALL)
1350 if m_thumb is not None:
1351 video_thumbnail = m_thumb.group(1)
1352 elif 'thumbnail_url' not in video_info:
1353 self._downloader.report_warning(u'unable to extract video thumbnail')
1354 video_thumbnail = None
1355 else: # don't panic if we can't find it
1356 video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
1357
1358 # upload date
1359 upload_date = None
1360 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
1361 if mobj is not None:
1362 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1363 upload_date = unified_strdate(upload_date)
1364
1365 # description
1366 video_description = get_element_by_id("eow-description", video_webpage)
1367 if video_description:
1368 video_description = clean_html(video_description)
1369 else:
1370 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1371 if fd_mobj:
1372 video_description = unescapeHTML(fd_mobj.group(1))
1373 else:
1374 video_description = u''
1375
1376 # subtitles
1377 video_subtitles = self.extract_subtitles(video_id, video_webpage)
1378
1379 if self._downloader.params.get('listsubtitles', False):
1380 self._list_available_subtitles(video_id, video_webpage)
1381 return
1382
1383 if 'length_seconds' not in video_info:
1384 self._downloader.report_warning(u'unable to extract video duration')
1385 video_duration = ''
1386 else:
1387 video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
1388
1389 # annotations
1390 video_annotations = None
1391 if self._downloader.params.get('writeannotations', False):
1392 video_annotations = self._extract_annotations(video_id)
1393
1394 # Decide which formats to download
1395
1396 try:
1397 mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
1398 if not mobj:
1399 raise ValueError('Could not find vevo ID')
1400 info = json.loads(mobj.group(1))
1401 args = info['args']
1402 # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
1403 # this signatures are encrypted
1404 if 'url_encoded_fmt_stream_map' not in args:
1405 raise ValueError(u'No stream_map present') # caught below
1406 m_s = re.search(r'[&,]s=', args['url_encoded_fmt_stream_map'])
1407 if m_s is not None:
1408 self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
1409 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
1410 m_s = re.search(r'[&,]s=', args.get('adaptive_fmts', u''))
1411 if m_s is not None:
1412 if 'url_encoded_fmt_stream_map' in video_info:
1413 video_info['url_encoded_fmt_stream_map'][0] += ',' + args['adaptive_fmts']
1414 else:
1415 video_info['url_encoded_fmt_stream_map'] = [args['adaptive_fmts']]
1416 elif 'adaptive_fmts' in video_info:
1417 if 'url_encoded_fmt_stream_map' in video_info:
1418 video_info['url_encoded_fmt_stream_map'][0] += ',' + video_info['adaptive_fmts'][0]
1419 else:
1420 video_info['url_encoded_fmt_stream_map'] = video_info['adaptive_fmts']
1421 except ValueError:
1422 pass
1423
1424 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1425 self.report_rtmp_download()
1426 video_url_list = [(None, video_info['conn'][0])]
1427 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
1428 if 'rtmpe%3Dyes' in video_info['url_encoded_fmt_stream_map'][0]:
1429 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1430 url_map = {}
1431 for url_data_str in video_info['url_encoded_fmt_stream_map'][0].split(','):
1432 url_data = compat_parse_qs(url_data_str)
1433 if 'itag' in url_data and 'url' in url_data:
1434 url = url_data['url'][0]
1435 if 'sig' in url_data:
1436 url += '&signature=' + url_data['sig'][0]
1437 elif 's' in url_data:
1438 encrypted_sig = url_data['s'][0]
1439 if self._downloader.params.get('verbose'):
1440 if age_gate:
1441 if player_url is None:
1442 player_version = 'unknown'
1443 else:
1444 player_version = self._search_regex(
1445 r'-(.+)\.swf$', player_url,
1446 u'flash player', fatal=False)
1447 player_desc = 'flash player %s' % player_version
1448 else:
1449 player_version = self._search_regex(
1450 r'html5player-(.+?)\.js', video_webpage,
1451 'html5 player', fatal=False)
1452 player_desc = u'html5 player %s' % player_version
1453
1454 parts_sizes = u'.'.join(compat_str(len(part)) for part in encrypted_sig.split('.'))
1455 self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' %
1456 (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc))
1457
1458 if not age_gate:
1459 jsplayer_url_json = self._search_regex(
1460 r'"assets":.+?"js":\s*("[^"]+")',
1461 video_webpage, u'JS player URL')
1462 player_url = json.loads(jsplayer_url_json)
1463
1464 signature = self._decrypt_signature(
1465 encrypted_sig, video_id, player_url, age_gate)
1466 url += '&signature=' + signature
1467 if 'ratebypass' not in url:
1468 url += '&ratebypass=yes'
1469 url_map[url_data['itag'][0]] = url
1470 video_url_list = self._get_video_url_list(url_map)
1471 if not video_url_list:
1472 return
1473 elif video_info.get('hlsvp'):
1474 manifest_url = video_info['hlsvp'][0]
1475 url_map = self._extract_from_m3u8(manifest_url, video_id)
1476 video_url_list = self._get_video_url_list(url_map)
1477 if not video_url_list:
1478 return
1479
1480 else:
1481 raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1482
1483 results = []
1484 for format_param, video_real_url in video_url_list:
1485 # Extension
1486 video_extension = self._video_extensions.get(format_param, 'flv')
1487
1488 video_format = '{0} - {1}{2}'.format(format_param if format_param else video_extension,
1489 self._video_dimensions.get(format_param, '???'),
1490 ' ('+self._special_itags[format_param]+')' if format_param in self._special_itags else '')
1491
1492 results.append({
1493 'id': video_id,
1494 'url': video_real_url,
1495 'uploader': video_uploader,
1496 'uploader_id': video_uploader_id,
1497 'upload_date': upload_date,
1498 'title': video_title,
1499 'ext': video_extension,
1500 'format': video_format,
1501 'thumbnail': video_thumbnail,
1502 'description': video_description,
1503 'player_url': player_url,
1504 'subtitles': video_subtitles,
1505 'duration': video_duration,
1506 'age_limit': 18 if age_gate else 0,
1507 'annotations': video_annotations
1508 })
1509 return results
1510
1511 class YoutubePlaylistIE(InfoExtractor):
1512 IE_DESC = u'YouTube.com playlists'
1513 _VALID_URL = r"""(?:
1514 (?:https?://)?
1515 (?:\w+\.)?
1516 youtube\.com/
1517 (?:
1518 (?:course|view_play_list|my_playlists|artist|playlist|watch)
1519 \? (?:.*?&)*? (?:p|a|list)=
1520 | p/
1521 )
1522 ((?:PL|EC|UU|FL)?[0-9A-Za-z-_]{10,})
1523 .*
1524 |
1525 ((?:PL|EC|UU|FL)[0-9A-Za-z-_]{10,})
1526 )"""
1527 _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/playlists/%s?max-results=%i&start-index=%i&v=2&alt=json&safeSearch=none'
1528 _MAX_RESULTS = 50
1529 IE_NAME = u'youtube:playlist'
1530
1531 @classmethod
1532 def suitable(cls, url):
1533 """Receives a URL and returns True if suitable for this IE."""
1534 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
1535
1536 def _real_extract(self, url):
1537 # Extract playlist id
1538 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1539 if mobj is None:
1540 raise ExtractorError(u'Invalid URL: %s' % url)
1541 playlist_id = mobj.group(1) or mobj.group(2)
1542
1543 # Check if it's a video-specific URL
1544 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
1545 if 'v' in query_dict:
1546 video_id = query_dict['v'][0]
1547 if self._downloader.params.get('noplaylist'):
1548 self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id)
1549 return self.url_result('https://www.youtube.com/watch?v=' + video_id, 'Youtube')
1550 else:
1551 self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id))
1552
1553 # Download playlist videos from API
1554 videos = []
1555
1556 for page_num in itertools.count(1):
1557 start_index = self._MAX_RESULTS * (page_num - 1) + 1
1558 if start_index >= 1000:
1559 self._downloader.report_warning(u'Max number of results reached')
1560 break
1561 url = self._TEMPLATE_URL % (playlist_id, self._MAX_RESULTS, start_index)
1562 page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
1563
1564 try:
1565 response = json.loads(page)
1566 except ValueError as err:
1567 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
1568
1569 if 'feed' not in response:
1570 raise ExtractorError(u'Got a malformed response from YouTube API')
1571 playlist_title = response['feed']['title']['$t']
1572 if 'entry' not in response['feed']:
1573 # Number of videos is a multiple of self._MAX_RESULTS
1574 break
1575
1576 for entry in response['feed']['entry']:
1577 index = entry['yt$position']['$t']
1578 if 'media$group' in entry and 'yt$videoid' in entry['media$group']:
1579 videos.append((
1580 index,
1581 'https://www.youtube.com/watch?v=' + entry['media$group']['yt$videoid']['$t']
1582 ))
1583
1584 videos = [v[1] for v in sorted(videos)]
1585
1586 url_results = [self.url_result(vurl, 'Youtube') for vurl in videos]
1587 return [self.playlist_result(url_results, playlist_id, playlist_title)]
1588
1589
1590 class YoutubeChannelIE(InfoExtractor):
1591 IE_DESC = u'YouTube.com channels'
1592 _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
1593 _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
1594 _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
1595 _MORE_PAGES_URL = 'http://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
1596 IE_NAME = u'youtube:channel'
1597
1598 def extract_videos_from_page(self, page):
1599 ids_in_page = []
1600 for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
1601 if mobj.group(1) not in ids_in_page:
1602 ids_in_page.append(mobj.group(1))
1603 return ids_in_page
1604
1605 def _real_extract(self, url):
1606 # Extract channel id
1607 mobj = re.match(self._VALID_URL, url)
1608 if mobj is None:
1609 raise ExtractorError(u'Invalid URL: %s' % url)
1610
1611 # Download channel page
1612 channel_id = mobj.group(1)
1613 video_ids = []
1614 pagenum = 1
1615
1616 url = self._TEMPLATE_URL % (channel_id, pagenum)
1617 page = self._download_webpage(url, channel_id,
1618 u'Downloading page #%s' % pagenum)
1619
1620 # Extract video identifiers
1621 ids_in_page = self.extract_videos_from_page(page)
1622 video_ids.extend(ids_in_page)
1623
1624 # Download any subsequent channel pages using the json-based channel_ajax query
1625 if self._MORE_PAGES_INDICATOR in page:
1626 for pagenum in itertools.count(1):
1627 url = self._MORE_PAGES_URL % (pagenum, channel_id)
1628 page = self._download_webpage(url, channel_id,
1629 u'Downloading page #%s' % pagenum)
1630
1631 page = json.loads(page)
1632
1633 ids_in_page = self.extract_videos_from_page(page['content_html'])
1634 video_ids.extend(ids_in_page)
1635
1636 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
1637 break
1638
1639 self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
1640
1641 urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids]
1642 url_entries = [self.url_result(eurl, 'Youtube') for eurl in urls]
1643 return [self.playlist_result(url_entries, channel_id)]
1644
1645
1646 class YoutubeUserIE(InfoExtractor):
1647 IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
1648 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
1649 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1650 _GDATA_PAGE_SIZE = 50
1651 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
1652 IE_NAME = u'youtube:user'
1653
1654 @classmethod
1655 def suitable(cls, url):
1656 # Don't return True if the url can be extracted with other youtube
1657 # extractor, the regex would is too permissive and it would match.
1658 other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
1659 if any(ie.suitable(url) for ie in other_ies): return False
1660 else: return super(YoutubeUserIE, cls).suitable(url)
1661
1662 def _real_extract(self, url):
1663 # Extract username
1664 mobj = re.match(self._VALID_URL, url)
1665 if mobj is None:
1666 raise ExtractorError(u'Invalid URL: %s' % url)
1667
1668 username = mobj.group(1)
1669
1670 # Download video ids using YouTube Data API. Result size per
1671 # query is limited (currently to 50 videos) so we need to query
1672 # page by page until there are no video ids - it means we got
1673 # all of them.
1674
1675 video_ids = []
1676
1677 for pagenum in itertools.count(0):
1678 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1679
1680 gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
1681 page = self._download_webpage(gdata_url, username,
1682 u'Downloading video ids from %d to %d' % (start_index, start_index + self._GDATA_PAGE_SIZE))
1683
1684 try:
1685 response = json.loads(page)
1686 except ValueError as err:
1687 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
1688 if 'entry' not in response['feed']:
1689 # Number of videos is a multiple of self._MAX_RESULTS
1690 break
1691
1692 # Extract video identifiers
1693 ids_in_page = []
1694 for entry in response['feed']['entry']:
1695 ids_in_page.append(entry['id']['$t'].split('/')[-1])
1696 video_ids.extend(ids_in_page)
1697
1698 # A little optimization - if current page is not
1699 # "full", ie. does not contain PAGE_SIZE video ids then
1700 # we can assume that this page is the last one - there
1701 # are no more ids on further pages - no need to query
1702 # again.
1703
1704 if len(ids_in_page) < self._GDATA_PAGE_SIZE:
1705 break
1706
1707 urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids]
1708 url_results = [self.url_result(rurl, 'Youtube') for rurl in urls]
1709 return [self.playlist_result(url_results, playlist_title = username)]
1710
1711 class YoutubeSearchIE(SearchInfoExtractor):
1712 IE_DESC = u'YouTube.com searches'
1713 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1714 _MAX_RESULTS = 1000
1715 IE_NAME = u'youtube:search'
1716 _SEARCH_KEY = 'ytsearch'
1717
1718 def report_download_page(self, query, pagenum):
1719 """Report attempt to download search page with given number."""
1720 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1721
1722 def _get_n_results(self, query, n):
1723 """Get a specified number of results for a query"""
1724
1725 video_ids = []
1726 pagenum = 0
1727 limit = n
1728
1729 while (50 * pagenum) < limit:
1730 self.report_download_page(query, pagenum+1)
1731 result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
1732 request = compat_urllib_request.Request(result_url)
1733 try:
1734 data = compat_urllib_request.urlopen(request).read().decode('utf-8')
1735 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1736 raise ExtractorError(u'Unable to download API page: %s' % compat_str(err))
1737 api_response = json.loads(data)['data']
1738
1739 if not 'items' in api_response:
1740 raise ExtractorError(u'[youtube] No video results')
1741
1742 new_ids = list(video['id'] for video in api_response['items'])
1743 video_ids += new_ids
1744
1745 limit = min(n, api_response['totalItems'])
1746 pagenum += 1
1747
1748 if len(video_ids) > n:
1749 video_ids = video_ids[:n]
1750 videos = [self.url_result('http://www.youtube.com/watch?v=%s' % id, 'Youtube') for id in video_ids]
1751 return self.playlist_result(videos, query)
1752
1753
1754 class YoutubeShowIE(InfoExtractor):
1755 IE_DESC = u'YouTube.com (multi-season) shows'
1756 _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
1757 IE_NAME = u'youtube:show'
1758
1759 def _real_extract(self, url):
1760 mobj = re.match(self._VALID_URL, url)
1761 show_name = mobj.group(1)
1762 webpage = self._download_webpage(url, show_name, u'Downloading show webpage')
1763 # There's one playlist for each season of the show
1764 m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
1765 self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
1766 return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
1767
1768
1769 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
1770 """
1771 Base class for extractors that fetch info from
1772 http://www.youtube.com/feed_ajax
1773 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
1774 """
1775 _LOGIN_REQUIRED = True
1776 _PAGING_STEP = 30
1777 # use action_load_personal_feed instead of action_load_system_feed
1778 _PERSONAL_FEED = False
1779
1780 @property
1781 def _FEED_TEMPLATE(self):
1782 action = 'action_load_system_feed'
1783 if self._PERSONAL_FEED:
1784 action = 'action_load_personal_feed'
1785 return 'http://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
1786
1787 @property
1788 def IE_NAME(self):
1789 return u'youtube:%s' % self._FEED_NAME
1790
1791 def _real_initialize(self):
1792 self._login()
1793
1794 def _real_extract(self, url):
1795 feed_entries = []
1796 # The step argument is available only in 2.7 or higher
1797 for i in itertools.count(0):
1798 paging = i*self._PAGING_STEP
1799 info = self._download_webpage(self._FEED_TEMPLATE % paging,
1800 u'%s feed' % self._FEED_NAME,
1801 u'Downloading page %s' % i)
1802 info = json.loads(info)
1803 feed_html = info['feed_html']
1804 m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
1805 ids = orderedSet(m.group(1) for m in m_ids)
1806 feed_entries.extend(self.url_result(id, 'Youtube') for id in ids)
1807 if info['paging'] is None:
1808 break
1809 return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
1810
1811 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
1812 IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword(requires authentication)'
1813 _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
1814 _FEED_NAME = 'subscriptions'
1815 _PLAYLIST_TITLE = u'Youtube Subscriptions'
1816
1817 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
1818 IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
1819 _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
1820 _FEED_NAME = 'recommended'
1821 _PLAYLIST_TITLE = u'Youtube Recommended videos'
1822
1823 class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
1824 IE_DESC = u'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
1825 _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
1826 _FEED_NAME = 'watch_later'
1827 _PLAYLIST_TITLE = u'Youtube Watch Later'
1828 _PAGING_STEP = 100
1829 _PERSONAL_FEED = True
1830
1831 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
1832 IE_NAME = u'youtube:favorites'
1833 IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'
1834 _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
1835 _LOGIN_REQUIRED = True
1836
1837 def _real_extract(self, url):
1838 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
1839 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, u'favourites playlist id')
1840 return self.url_result(playlist_id, 'YoutubePlaylist')
1841
1842
1843 class YoutubeTruncatedURLIE(InfoExtractor):
1844 IE_NAME = 'youtube:truncated_url'
1845 IE_DESC = False # Do not list
1846 _VALID_URL = r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$'
1847
1848 def _real_extract(self, url):
1849 raise ExtractorError(
1850 u'Did you forget to quote the URL? Remember that & is a meta '
1851 u'character in most shells, so you want to put the URL in quotes, '
1852 u'like youtube-dl '
1853 u'\'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\''
1854 u' (or simply youtube-dl BaW_jenozKc ).',
1855 expected=True)
Backup clone of https://github.com/ytdl-org unavailable since RIAA fucking raid...