]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/youtube.py
Update upstream source from tag 'upstream/2020.03.24'
[youtubedl] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 from __future__ import unicode_literals
4
5
6 import itertools
7 import json
8 import os.path
9 import random
10 import re
11 import time
12 import traceback
13
14 from .common import InfoExtractor, SearchInfoExtractor
15 from ..jsinterp import JSInterpreter
16 from ..swfinterp import SWFInterpreter
17 from ..compat import (
18 compat_chr,
19 compat_HTTPError,
20 compat_kwargs,
21 compat_parse_qs,
22 compat_urllib_parse_unquote,
23 compat_urllib_parse_unquote_plus,
24 compat_urllib_parse_urlencode,
25 compat_urllib_parse_urlparse,
26 compat_urlparse,
27 compat_str,
28 )
29 from ..utils import (
30 bool_or_none,
31 clean_html,
32 error_to_compat_str,
33 extract_attributes,
34 ExtractorError,
35 float_or_none,
36 get_element_by_attribute,
37 get_element_by_id,
38 int_or_none,
39 mimetype2ext,
40 orderedSet,
41 parse_codecs,
42 parse_duration,
43 remove_quotes,
44 remove_start,
45 smuggle_url,
46 str_or_none,
47 str_to_int,
48 try_get,
49 unescapeHTML,
50 unified_strdate,
51 unsmuggle_url,
52 uppercase_escape,
53 url_or_none,
54 urlencode_postdata,
55 )
56
57
58 class YoutubeBaseInfoExtractor(InfoExtractor):
59 """Provide base functions for Youtube extractors"""
60 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
61 _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
62
63 _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
64 _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
65 _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
66
67 _NETRC_MACHINE = 'youtube'
68 # If True it will raise an error if no login info is provided
69 _LOGIN_REQUIRED = False
70
71 _PLAYLIST_ID_RE = r'(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}'
72
73 def _set_language(self):
74 self._set_cookie(
75 '.youtube.com', 'PREF', 'f1=50000000&hl=en',
76 # YouTube sets the expire time to about two months
77 expire_time=time.time() + 2 * 30 * 24 * 3600)
78
79 def _ids_to_results(self, ids):
80 return [
81 self.url_result(vid_id, 'Youtube', video_id=vid_id)
82 for vid_id in ids]
83
84 def _login(self):
85 """
86 Attempt to log in to YouTube.
87 True is returned if successful or skipped.
88 False is returned if login failed.
89
90 If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
91 """
92 username, password = self._get_login_info()
93 # No authentication to be performed
94 if username is None:
95 if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
96 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
97 return True
98
99 login_page = self._download_webpage(
100 self._LOGIN_URL, None,
101 note='Downloading login page',
102 errnote='unable to fetch login page', fatal=False)
103 if login_page is False:
104 return
105
106 login_form = self._hidden_inputs(login_page)
107
108 def req(url, f_req, note, errnote):
109 data = login_form.copy()
110 data.update({
111 'pstMsg': 1,
112 'checkConnection': 'youtube',
113 'checkedDomains': 'youtube',
114 'hl': 'en',
115 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
116 'f.req': json.dumps(f_req),
117 'flowName': 'GlifWebSignIn',
118 'flowEntry': 'ServiceLogin',
119 # TODO: reverse actual botguard identifier generation algo
120 'bgRequest': '["identifier",""]',
121 })
122 return self._download_json(
123 url, None, note=note, errnote=errnote,
124 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
125 fatal=False,
126 data=urlencode_postdata(data), headers={
127 'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
128 'Google-Accounts-XSRF': 1,
129 })
130
131 def warn(message):
132 self._downloader.report_warning(message)
133
134 lookup_req = [
135 username,
136 None, [], None, 'US', None, None, 2, False, True,
137 [
138 None, None,
139 [2, 1, None, 1,
140 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
141 None, [], 4],
142 1, [None, None, []], None, None, None, True
143 ],
144 username,
145 ]
146
147 lookup_results = req(
148 self._LOOKUP_URL, lookup_req,
149 'Looking up account info', 'Unable to look up account info')
150
151 if lookup_results is False:
152 return False
153
154 user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
155 if not user_hash:
156 warn('Unable to extract user hash')
157 return False
158
159 challenge_req = [
160 user_hash,
161 None, 1, None, [1, None, None, None, [password, None, True]],
162 [
163 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
164 1, [None, None, []], None, None, None, True
165 ]]
166
167 challenge_results = req(
168 self._CHALLENGE_URL, challenge_req,
169 'Logging in', 'Unable to log in')
170
171 if challenge_results is False:
172 return
173
174 login_res = try_get(challenge_results, lambda x: x[0][5], list)
175 if login_res:
176 login_msg = try_get(login_res, lambda x: x[5], compat_str)
177 warn(
178 'Unable to login: %s' % 'Invalid password'
179 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
180 return False
181
182 res = try_get(challenge_results, lambda x: x[0][-1], list)
183 if not res:
184 warn('Unable to extract result entry')
185 return False
186
187 login_challenge = try_get(res, lambda x: x[0][0], list)
188 if login_challenge:
189 challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
190 if challenge_str == 'TWO_STEP_VERIFICATION':
191 # SEND_SUCCESS - TFA code has been successfully sent to phone
192 # QUOTA_EXCEEDED - reached the limit of TFA codes
193 status = try_get(login_challenge, lambda x: x[5], compat_str)
194 if status == 'QUOTA_EXCEEDED':
195 warn('Exceeded the limit of TFA codes, try later')
196 return False
197
198 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
199 if not tl:
200 warn('Unable to extract TL')
201 return False
202
203 tfa_code = self._get_tfa_info('2-step verification code')
204
205 if not tfa_code:
206 warn(
207 'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
208 '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
209 return False
210
211 tfa_code = remove_start(tfa_code, 'G-')
212
213 tfa_req = [
214 user_hash, None, 2, None,
215 [
216 9, None, None, None, None, None, None, None,
217 [None, tfa_code, True, 2]
218 ]]
219
220 tfa_results = req(
221 self._TFA_URL.format(tl), tfa_req,
222 'Submitting TFA code', 'Unable to submit TFA code')
223
224 if tfa_results is False:
225 return False
226
227 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
228 if tfa_res:
229 tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
230 warn(
231 'Unable to finish TFA: %s' % 'Invalid TFA code'
232 if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
233 return False
234
235 check_cookie_url = try_get(
236 tfa_results, lambda x: x[0][-1][2], compat_str)
237 else:
238 CHALLENGES = {
239 'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
240 'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
241 'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
242 }
243 challenge = CHALLENGES.get(
244 challenge_str,
245 '%s returned error %s.' % (self.IE_NAME, challenge_str))
246 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
247 return False
248 else:
249 check_cookie_url = try_get(res, lambda x: x[2], compat_str)
250
251 if not check_cookie_url:
252 warn('Unable to extract CheckCookie URL')
253 return False
254
255 check_cookie_results = self._download_webpage(
256 check_cookie_url, None, 'Checking cookie', fatal=False)
257
258 if check_cookie_results is False:
259 return False
260
261 if 'https://myaccount.google.com/' not in check_cookie_results:
262 warn('Unable to log in')
263 return False
264
265 return True
266
267 def _download_webpage_handle(self, *args, **kwargs):
268 query = kwargs.get('query', {}).copy()
269 query['disable_polymer'] = 'true'
270 kwargs['query'] = query
271 return super(YoutubeBaseInfoExtractor, self)._download_webpage_handle(
272 *args, **compat_kwargs(kwargs))
273
274 def _real_initialize(self):
275 if self._downloader is None:
276 return
277 self._set_language()
278 if not self._login():
279 return
280
281
282 class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
283 # Extract entries from page with "Load more" button
284 def _entries(self, page, playlist_id):
285 more_widget_html = content_html = page
286 for page_num in itertools.count(1):
287 for entry in self._process_page(content_html):
288 yield entry
289
290 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
291 if not mobj:
292 break
293
294 count = 0
295 retries = 3
296 while count <= retries:
297 try:
298 # Downloading page may result in intermittent 5xx HTTP error
299 # that is usually worked around with a retry
300 more = self._download_json(
301 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
302 'Downloading page #%s%s'
303 % (page_num, ' (retry #%d)' % count if count else ''),
304 transform_source=uppercase_escape)
305 break
306 except ExtractorError as e:
307 if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
308 count += 1
309 if count <= retries:
310 continue
311 raise
312
313 content_html = more['content_html']
314 if not content_html.strip():
315 # Some webpages show a "Load more" button but they don't
316 # have more videos
317 break
318 more_widget_html = more['load_more_widget_html']
319
320
321 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
322 def _process_page(self, content):
323 for video_id, video_title in self.extract_videos_from_page(content):
324 yield self.url_result(video_id, 'Youtube', video_id, video_title)
325
326 def extract_videos_from_page_impl(self, video_re, page, ids_in_page, titles_in_page):
327 for mobj in re.finditer(video_re, page):
328 # The link with index 0 is not the first video of the playlist (not sure if still actual)
329 if 'index' in mobj.groupdict() and mobj.group('id') == '0':
330 continue
331 video_id = mobj.group('id')
332 video_title = unescapeHTML(
333 mobj.group('title')) if 'title' in mobj.groupdict() else None
334 if video_title:
335 video_title = video_title.strip()
336 if video_title == 'ā–ŗ Play all':
337 video_title = None
338 try:
339 idx = ids_in_page.index(video_id)
340 if video_title and not titles_in_page[idx]:
341 titles_in_page[idx] = video_title
342 except ValueError:
343 ids_in_page.append(video_id)
344 titles_in_page.append(video_title)
345
346 def extract_videos_from_page(self, page):
347 ids_in_page = []
348 titles_in_page = []
349 self.extract_videos_from_page_impl(
350 self._VIDEO_RE, page, ids_in_page, titles_in_page)
351 return zip(ids_in_page, titles_in_page)
352
353
354 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
355 def _process_page(self, content):
356 for playlist_id in orderedSet(re.findall(
357 r'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
358 content)):
359 yield self.url_result(
360 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
361
362 def _real_extract(self, url):
363 playlist_id = self._match_id(url)
364 webpage = self._download_webpage(url, playlist_id)
365 title = self._og_search_title(webpage, fatal=False)
366 return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
367
368
369 class YoutubeIE(YoutubeBaseInfoExtractor):
370 IE_DESC = 'YouTube.com'
371 _VALID_URL = r"""(?x)^
372 (
373 (?:https?://|//) # http(s):// or protocol-independent URL
374 (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com/|
375 (?:www\.)?deturl\.com/www\.youtube\.com/|
376 (?:www\.)?pwnyoutube\.com/|
377 (?:www\.)?hooktube\.com/|
378 (?:www\.)?yourepeat\.com/|
379 tube\.majestyc\.net/|
380 # Invidious instances taken from https://github.com/omarroth/invidious/wiki/Invidious-Instances
381 (?:(?:www|dev)\.)?invidio\.us/|
382 (?:(?:www|no)\.)?invidiou\.sh/|
383 (?:(?:www|fi|de)\.)?invidious\.snopyta\.org/|
384 (?:www\.)?invidious\.kabi\.tk/|
385 (?:www\.)?invidious\.13ad\.de/|
386 (?:www\.)?invidious\.mastodon\.host/|
387 (?:www\.)?invidious\.nixnet\.xyz/|
388 (?:www\.)?invidious\.drycat\.fr/|
389 (?:www\.)?tube\.poal\.co/|
390 (?:www\.)?vid\.wxzm\.sx/|
391 (?:www\.)?yt\.elukerio\.org/|
392 (?:www\.)?yt\.lelux\.fi/|
393 (?:www\.)?kgg2m7yk5aybusll\.onion/|
394 (?:www\.)?qklhadlycap4cnod\.onion/|
395 (?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion/|
396 (?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion/|
397 (?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion/|
398 (?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion/|
399 (?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p/|
400 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
401 (?:.*?\#/)? # handle anchor (#/) redirect urls
402 (?: # the various things that can precede the ID:
403 (?:(?:v|embed|e)/(?!videoseries)) # v/ or embed/ or e/
404 |(?: # or the v= param in all its forms
405 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
406 (?:\?|\#!?) # the params delimiter ? or # or #!
407 (?:.*?[&;])?? # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
408 v=
409 )
410 ))
411 |(?:
412 youtu\.be| # just youtu.be/xxxx
413 vid\.plus| # or vid.plus/xxxx
414 zwearz\.com/watch| # or zwearz.com/watch/xxxx
415 )/
416 |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
417 )
418 )? # all until now is optional -> you can pass the naked ID
419 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
420 (?!.*?\blist=
421 (?:
422 %(playlist_id)s| # combined list/video URLs are handled by the playlist IE
423 WL # WL are handled by the watch later IE
424 )
425 )
426 (?(1).+)? # if we found the ID, everything can follow
427 $""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
428 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
429 _formats = {
430 '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
431 '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
432 '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
433 '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
434 '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
435 '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
436 '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
437 '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
438 # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
439 '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
440 '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
441 '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
442 '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
443 '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
444 '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
445 '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
446 '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
447 '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
448
449
450 # 3D videos
451 '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
452 '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
453 '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
454 '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
455 '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
456 '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
457 '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
458
459 # Apple HTTP Live Streaming
460 '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
461 '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
462 '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
463 '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
464 '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
465 '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
466 '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
467 '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
468
469 # DASH mp4 video
470 '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
471 '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
472 '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
473 '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
474 '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
475 '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
476 '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
477 '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
478 '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
479 '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
480 '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
481 '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
482
483 # Dash mp4 audio
484 '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
485 '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
486 '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
487 '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
488 '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
489 '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
490 '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
491
492 # Dash webm
493 '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
494 '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
495 '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
496 '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
497 '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
498 '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
499 '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
500 '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
501 '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
502 '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
503 '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
504 '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
505 '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
506 '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
507 '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
508 # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
509 '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
510 '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
511 '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
512 '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
513 '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
514 '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
515
516 # Dash webm audio
517 '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
518 '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
519
520 # Dash webm audio with opus inside
521 '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
522 '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
523 '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
524
525 # RTMP (unnamed)
526 '_rtmp': {'protocol': 'rtmp'},
527
528 # av01 video only formats sometimes served with "unknown" codecs
529 '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
530 '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
531 '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
532 '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
533 }
534 _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
535
536 _GEO_BYPASS = False
537
538 IE_NAME = 'youtube'
539 _TESTS = [
540 {
541 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
542 'info_dict': {
543 'id': 'BaW_jenozKc',
544 'ext': 'mp4',
545 'title': 'youtube-dl test video "\'/\\Ƥā†­š•',
546 'uploader': 'Philipp Hagemeister',
547 'uploader_id': 'phihag',
548 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
549 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
550 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
551 'upload_date': '20121002',
552 'description': 'test chars: "\'/\\Ƥā†­š•\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
553 'categories': ['Science & Technology'],
554 'tags': ['youtube-dl'],
555 'duration': 10,
556 'view_count': int,
557 'like_count': int,
558 'dislike_count': int,
559 'start_time': 1,
560 'end_time': 9,
561 }
562 },
563 {
564 'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY',
565 'note': 'Test generic use_cipher_signature video (#897)',
566 'info_dict': {
567 'id': 'UxxajLWwzqY',
568 'ext': 'mp4',
569 'upload_date': '20120506',
570 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
571 'alt_title': 'I Love It (feat. Charli XCX)',
572 'description': 'md5:19a2f98d9032b9311e686ed039564f63',
573 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
574 'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
575 'iconic ep', 'iconic', 'love', 'it'],
576 'duration': 180,
577 'uploader': 'Icona Pop',
578 'uploader_id': 'IconaPop',
579 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop',
580 'creator': 'Icona Pop',
581 'track': 'I Love It (feat. Charli XCX)',
582 'artist': 'Icona Pop',
583 }
584 },
585 {
586 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
587 'note': 'Test VEVO video with age protection (#956)',
588 'info_dict': {
589 'id': '07FYdnEawAQ',
590 'ext': 'mp4',
591 'upload_date': '20130703',
592 'title': 'Justin Timberlake - Tunnel Vision (Official Music Video) (Explicit)',
593 'alt_title': 'Tunnel Vision',
594 'description': 'md5:07dab3356cde4199048e4c7cd93471e1',
595 'duration': 419,
596 'uploader': 'justintimberlakeVEVO',
597 'uploader_id': 'justintimberlakeVEVO',
598 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
599 'creator': 'Justin Timberlake',
600 'track': 'Tunnel Vision',
601 'artist': 'Justin Timberlake',
602 'age_limit': 18,
603 }
604 },
605 {
606 'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
607 'note': 'Embed-only video (#1746)',
608 'info_dict': {
609 'id': 'yZIXLfi8CZQ',
610 'ext': 'mp4',
611 'upload_date': '20120608',
612 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
613 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
614 'uploader': 'SET India',
615 'uploader_id': 'setindia',
616 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
617 'age_limit': 18,
618 }
619 },
620 {
621 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
622 'note': 'Use the first video ID in the URL',
623 'info_dict': {
624 'id': 'BaW_jenozKc',
625 'ext': 'mp4',
626 'title': 'youtube-dl test video "\'/\\Ƥā†­š•',
627 'uploader': 'Philipp Hagemeister',
628 'uploader_id': 'phihag',
629 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
630 'upload_date': '20121002',
631 'description': 'test chars: "\'/\\Ƥā†­š•\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
632 'categories': ['Science & Technology'],
633 'tags': ['youtube-dl'],
634 'duration': 10,
635 'view_count': int,
636 'like_count': int,
637 'dislike_count': int,
638 },
639 'params': {
640 'skip_download': True,
641 },
642 },
643 {
644 'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
645 'note': '256k DASH audio (format 141) via DASH manifest',
646 'info_dict': {
647 'id': 'a9LDPn-MO4I',
648 'ext': 'm4a',
649 'upload_date': '20121002',
650 'uploader_id': '8KVIDEO',
651 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
652 'description': '',
653 'uploader': '8KVIDEO',
654 'title': 'UHDTV TEST 8K VIDEO.mp4'
655 },
656 'params': {
657 'youtube_include_dash_manifest': True,
658 'format': '141',
659 },
660 'skip': 'format 141 not served anymore',
661 },
662 # DASH manifest with encrypted signature
663 {
664 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
665 'info_dict': {
666 'id': 'IB3lcPjvWLA',
667 'ext': 'm4a',
668 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
669 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
670 'duration': 244,
671 'uploader': 'AfrojackVEVO',
672 'uploader_id': 'AfrojackVEVO',
673 'upload_date': '20131011',
674 },
675 'params': {
676 'youtube_include_dash_manifest': True,
677 'format': '141/bestaudio[ext=m4a]',
678 },
679 },
680 # JS player signature function name containing $
681 {
682 'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
683 'info_dict': {
684 'id': 'nfWlot6h_JM',
685 'ext': 'm4a',
686 'title': 'Taylor Swift - Shake It Off',
687 'description': 'md5:307195cd21ff7fa352270fe884570ef0',
688 'duration': 242,
689 'uploader': 'TaylorSwiftVEVO',
690 'uploader_id': 'TaylorSwiftVEVO',
691 'upload_date': '20140818',
692 },
693 'params': {
694 'youtube_include_dash_manifest': True,
695 'format': '141/bestaudio[ext=m4a]',
696 },
697 },
698 # Controversy video
699 {
700 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
701 'info_dict': {
702 'id': 'T4XJQO3qol8',
703 'ext': 'mp4',
704 'duration': 219,
705 'upload_date': '20100909',
706 'uploader': 'Amazing Atheist',
707 'uploader_id': 'TheAmazingAtheist',
708 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
709 'title': 'Burning Everyone\'s Koran',
710 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
711 }
712 },
713 # Normal age-gate video (No vevo, embed allowed)
714 {
715 'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
716 'info_dict': {
717 'id': 'HtVdAasjOgU',
718 'ext': 'mp4',
719 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
720 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
721 'duration': 142,
722 'uploader': 'The Witcher',
723 'uploader_id': 'WitcherGame',
724 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
725 'upload_date': '20140605',
726 'age_limit': 18,
727 },
728 },
729 # Age-gate video with encrypted signature
730 {
731 'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
732 'info_dict': {
733 'id': '6kLq3WMV1nU',
734 'ext': 'mp4',
735 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
736 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
737 'duration': 246,
738 'uploader': 'LloydVEVO',
739 'uploader_id': 'LloydVEVO',
740 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
741 'upload_date': '20110629',
742 'age_limit': 18,
743 },
744 },
745 # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
746 # YouTube Red ad is not captured for creator
747 {
748 'url': '__2ABJjxzNo',
749 'info_dict': {
750 'id': '__2ABJjxzNo',
751 'ext': 'mp4',
752 'duration': 266,
753 'upload_date': '20100430',
754 'uploader_id': 'deadmau5',
755 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
756 'creator': 'Dada Life, deadmau5',
757 'description': 'md5:12c56784b8032162bb936a5f76d55360',
758 'uploader': 'deadmau5',
759 'title': 'Deadmau5 - Some Chords (HD)',
760 'alt_title': 'This Machine Kills Some Chords',
761 },
762 'expected_warnings': [
763 'DASH manifest missing',
764 ]
765 },
766 # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
767 {
768 'url': 'lqQg6PlCWgI',
769 'info_dict': {
770 'id': 'lqQg6PlCWgI',
771 'ext': 'mp4',
772 'duration': 6085,
773 'upload_date': '20150827',
774 'uploader_id': 'olympic',
775 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
776 'description': 'HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
777 'uploader': 'Olympic',
778 'title': 'Hockey - Women - GER-AUS - London 2012 Olympic Games',
779 },
780 'params': {
781 'skip_download': 'requires avconv',
782 }
783 },
784 # Non-square pixels
785 {
786 'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
787 'info_dict': {
788 'id': '_b-2C3KPAM0',
789 'ext': 'mp4',
790 'stretched_ratio': 16 / 9.,
791 'duration': 85,
792 'upload_date': '20110310',
793 'uploader_id': 'AllenMeow',
794 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
795 'description': 'made by Wacom from Korea | 字幕&åŠ ę²¹ę·»é†‹ by TY\'s Allen | ę„Ÿč¬heylisa00cavey1001同å­øē†±ęƒ…ęä¾›ę¢—åŠēæ»č­Æ',
796 'uploader': 'å­«į„‹į„…',
797 'title': '[A-made] č®Šę…‹å¦å­—å¹•ē‰ˆ å¤Ŗ妍 ęˆ‘å°±ę˜Æ這ęØ£ēš„äŗŗ',
798 },
799 },
800 # url_encoded_fmt_stream_map is empty string
801 {
802 'url': 'qEJwOuvDf7I',
803 'info_dict': {
804 'id': 'qEJwOuvDf7I',
805 'ext': 'webm',
806 'title': 'ŠžŠ±ŃŃƒŠ¶Š“ŠµŠ½ŠøŠµ суŠ“ŠµŠ±Š½Š¾Š¹ ŠæрŠ°ŠŗтŠøŠŗŠø ŠæŠ¾ Š²Ń‹Š±Š¾Ń€Š°Š¼ 14 сŠµŠ½Ń‚яŠ±Ń€Ń 2014 Š³Š¾Š“Š° Š² Š”Š°Š½Šŗт-ŠŸŠµŃ‚ŠµŃ€Š±ŃƒŃ€Š³Šµ',
807 'description': '',
808 'upload_date': '20150404',
809 'uploader_id': 'spbelect',
810 'uploader': 'ŠŠ°Š±Š»ŃŽŠ“Š°Ń‚ŠµŠ»Šø ŠŸŠµŃ‚ŠµŃ€Š±ŃƒŃ€Š³Š°',
811 },
812 'params': {
813 'skip_download': 'requires avconv',
814 },
815 'skip': 'This live event has ended.',
816 },
817 # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
818 {
819 'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
820 'info_dict': {
821 'id': 'FIl7x6_3R5Y',
822 'ext': 'webm',
823 'title': 'md5:7b81415841e02ecd4313668cde88737a',
824 'description': 'md5:116377fd2963b81ec4ce64b542173306',
825 'duration': 220,
826 'upload_date': '20150625',
827 'uploader_id': 'dorappi2000',
828 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
829 'uploader': 'dorappi2000',
830 'formats': 'mincount:31',
831 },
832 'skip': 'not actual anymore',
833 },
834 # DASH manifest with segment_list
835 {
836 'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
837 'md5': '8ce563a1d667b599d21064e982ab9e31',
838 'info_dict': {
839 'id': 'CsmdDsKjzN8',
840 'ext': 'mp4',
841 'upload_date': '20150501', # According to '<meta itemprop="datePublished"', but in other places it's 20150510
842 'uploader': 'Airtek',
843 'description': 'RetransmisiĆ³n en directo de la XVIII media maratĆ³n de Zaragoza.',
844 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
845 'title': 'RetransmisiĆ³n XVIII Media maratĆ³n Zaragoza 2015',
846 },
847 'params': {
848 'youtube_include_dash_manifest': True,
849 'format': '135', # bestvideo
850 },
851 'skip': 'This live event has ended.',
852 },
853 {
854 # Multifeed videos (multiple cameras), URL is for Main Camera
855 'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
856 'info_dict': {
857 'id': 'jqWvoWXjCVs',
858 'title': 'teamPGP: Rocket League Noob Stream',
859 'description': 'md5:dc7872fb300e143831327f1bae3af010',
860 },
861 'playlist': [{
862 'info_dict': {
863 'id': 'jqWvoWXjCVs',
864 'ext': 'mp4',
865 'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
866 'description': 'md5:dc7872fb300e143831327f1bae3af010',
867 'duration': 7335,
868 'upload_date': '20150721',
869 'uploader': 'Beer Games Beer',
870 'uploader_id': 'beergamesbeer',
871 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
872 'license': 'Standard YouTube License',
873 },
874 }, {
875 'info_dict': {
876 'id': '6h8e8xoXJzg',
877 'ext': 'mp4',
878 'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
879 'description': 'md5:dc7872fb300e143831327f1bae3af010',
880 'duration': 7337,
881 'upload_date': '20150721',
882 'uploader': 'Beer Games Beer',
883 'uploader_id': 'beergamesbeer',
884 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
885 'license': 'Standard YouTube License',
886 },
887 }, {
888 'info_dict': {
889 'id': 'PUOgX5z9xZw',
890 'ext': 'mp4',
891 'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
892 'description': 'md5:dc7872fb300e143831327f1bae3af010',
893 'duration': 7337,
894 'upload_date': '20150721',
895 'uploader': 'Beer Games Beer',
896 'uploader_id': 'beergamesbeer',
897 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
898 'license': 'Standard YouTube License',
899 },
900 }, {
901 'info_dict': {
902 'id': 'teuwxikvS5k',
903 'ext': 'mp4',
904 'title': 'teamPGP: Rocket League Noob Stream (zim)',
905 'description': 'md5:dc7872fb300e143831327f1bae3af010',
906 'duration': 7334,
907 'upload_date': '20150721',
908 'uploader': 'Beer Games Beer',
909 'uploader_id': 'beergamesbeer',
910 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
911 'license': 'Standard YouTube License',
912 },
913 }],
914 'params': {
915 'skip_download': True,
916 },
917 'skip': 'This video is not available.',
918 },
919 {
920 # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
921 'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
922 'info_dict': {
923 'id': 'gVfLd0zydlo',
924 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
925 },
926 'playlist_count': 2,
927 'skip': 'Not multifeed anymore',
928 },
929 {
930 'url': 'https://vid.plus/FlRa-iH7PGw',
931 'only_matching': True,
932 },
933 {
934 'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
935 'only_matching': True,
936 },
937 {
938 # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
939 # Also tests cut-off URL expansion in video description (see
940 # https://github.com/ytdl-org/youtube-dl/issues/1892,
941 # https://github.com/ytdl-org/youtube-dl/issues/8164)
942 'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
943 'info_dict': {
944 'id': 'lsguqyKfVQg',
945 'ext': 'mp4',
946 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
947 'alt_title': 'Dark Walk - Position Music',
948 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
949 'duration': 133,
950 'upload_date': '20151119',
951 'uploader_id': 'IronSoulElf',
952 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
953 'uploader': 'IronSoulElf',
954 'creator': 'Todd Haberman, Daniel Law Heath and Aaron Kaplan',
955 'track': 'Dark Walk - Position Music',
956 'artist': 'Todd Haberman, Daniel Law Heath and Aaron Kaplan',
957 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
958 },
959 'params': {
960 'skip_download': True,
961 },
962 },
963 {
964 # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
965 'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
966 'only_matching': True,
967 },
968 {
969 # Video with yt:stretch=17:0
970 'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
971 'info_dict': {
972 'id': 'Q39EVAstoRM',
973 'ext': 'mp4',
974 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
975 'description': 'md5:ee18a25c350637c8faff806845bddee9',
976 'upload_date': '20151107',
977 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
978 'uploader': 'CH GAMER DROID',
979 },
980 'params': {
981 'skip_download': True,
982 },
983 'skip': 'This video does not exist.',
984 },
985 {
986 # Video licensed under Creative Commons
987 'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
988 'info_dict': {
989 'id': 'M4gD1WSo5mA',
990 'ext': 'mp4',
991 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
992 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
993 'duration': 721,
994 'upload_date': '20150127',
995 'uploader_id': 'BerkmanCenter',
996 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
997 'uploader': 'The Berkman Klein Center for Internet & Society',
998 'license': 'Creative Commons Attribution license (reuse allowed)',
999 },
1000 'params': {
1001 'skip_download': True,
1002 },
1003 },
1004 {
1005 # Channel-like uploader_url
1006 'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1007 'info_dict': {
1008 'id': 'eQcmzGIKrzg',
1009 'ext': 'mp4',
1010 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1011 'description': 'md5:dda0d780d5a6e120758d1711d062a867',
1012 'duration': 4060,
1013 'upload_date': '20151119',
1014 'uploader': 'Bernie Sanders',
1015 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1016 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1017 'license': 'Creative Commons Attribution license (reuse allowed)',
1018 },
1019 'params': {
1020 'skip_download': True,
1021 },
1022 },
1023 {
1024 'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1025 'only_matching': True,
1026 },
1027 {
1028 # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1029 'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1030 'only_matching': True,
1031 },
1032 {
1033 # Rental video preview
1034 'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1035 'info_dict': {
1036 'id': 'uGpuVWrhIzE',
1037 'ext': 'mp4',
1038 'title': 'Piku - Trailer',
1039 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1040 'upload_date': '20150811',
1041 'uploader': 'FlixMatrix',
1042 'uploader_id': 'FlixMatrixKaravan',
1043 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1044 'license': 'Standard YouTube License',
1045 },
1046 'params': {
1047 'skip_download': True,
1048 },
1049 'skip': 'This video is not available.',
1050 },
1051 {
1052 # YouTube Red video with episode data
1053 'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1054 'info_dict': {
1055 'id': 'iqKdEhx-dD4',
1056 'ext': 'mp4',
1057 'title': 'Isolation - Mind Field (Ep 1)',
1058 'description': 'md5:46a29be4ceffa65b92d277b93f463c0f',
1059 'duration': 2085,
1060 'upload_date': '20170118',
1061 'uploader': 'Vsauce',
1062 'uploader_id': 'Vsauce',
1063 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1064 'series': 'Mind Field',
1065 'season_number': 1,
1066 'episode_number': 1,
1067 },
1068 'params': {
1069 'skip_download': True,
1070 },
1071 'expected_warnings': [
1072 'Skipping DASH manifest',
1073 ],
1074 },
1075 {
1076 # The following content has been identified by the YouTube community
1077 # as inappropriate or offensive to some audiences.
1078 'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1079 'info_dict': {
1080 'id': '6SJNVb0GnPI',
1081 'ext': 'mp4',
1082 'title': 'Race Differences in Intelligence',
1083 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1084 'duration': 965,
1085 'upload_date': '20140124',
1086 'uploader': 'New Century Foundation',
1087 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1088 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1089 },
1090 'params': {
1091 'skip_download': True,
1092 },
1093 },
1094 {
1095 # itag 212
1096 'url': '1t24XAntNCY',
1097 'only_matching': True,
1098 },
1099 {
1100 # geo restricted to JP
1101 'url': 'sJL6WA-aGkQ',
1102 'only_matching': True,
1103 },
1104 {
1105 'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
1106 'only_matching': True,
1107 },
1108 {
1109 'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1110 'only_matching': True,
1111 },
1112 {
1113 # DRM protected
1114 'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1115 'only_matching': True,
1116 },
1117 {
1118 # Video with unsupported adaptive stream type formats
1119 'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1120 'info_dict': {
1121 'id': 'Z4Vy8R84T1U',
1122 'ext': 'mp4',
1123 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1124 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1125 'duration': 433,
1126 'upload_date': '20130923',
1127 'uploader': 'Amelia Putri Harwita',
1128 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1129 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1130 'formats': 'maxcount:10',
1131 },
1132 'params': {
1133 'skip_download': True,
1134 'youtube_include_dash_manifest': False,
1135 },
1136 'skip': 'not actual anymore',
1137 },
1138 {
1139 # Youtube Music Auto-generated description
1140 'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1141 'info_dict': {
1142 'id': 'MgNrAu2pzNs',
1143 'ext': 'mp4',
1144 'title': 'Voyeur Girl',
1145 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1146 'upload_date': '20190312',
1147 'uploader': 'Stephen - Topic',
1148 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1149 'artist': 'Stephen',
1150 'track': 'Voyeur Girl',
1151 'album': 'it\'s too much love to know my dear',
1152 'release_date': '20190313',
1153 'release_year': 2019,
1154 },
1155 'params': {
1156 'skip_download': True,
1157 },
1158 },
1159 {
1160 # Youtube Music Auto-generated description
1161 # Retrieve 'artist' field from 'Artist:' in video description
1162 # when it is present on youtube music video
1163 'url': 'https://www.youtube.com/watch?v=k0jLE7tTwjY',
1164 'info_dict': {
1165 'id': 'k0jLE7tTwjY',
1166 'ext': 'mp4',
1167 'title': 'Latch Feat. Sam Smith',
1168 'description': 'md5:3cb1e8101a7c85fcba9b4fb41b951335',
1169 'upload_date': '20150110',
1170 'uploader': 'Various Artists - Topic',
1171 'uploader_id': 'UCNkEcmYdjrH4RqtNgh7BZ9w',
1172 'artist': 'Disclosure',
1173 'track': 'Latch Feat. Sam Smith',
1174 'album': 'Latch Featuring Sam Smith',
1175 'release_date': '20121008',
1176 'release_year': 2012,
1177 },
1178 'params': {
1179 'skip_download': True,
1180 },
1181 },
1182 {
1183 # Youtube Music Auto-generated description
1184 # handle multiple artists on youtube music video
1185 'url': 'https://www.youtube.com/watch?v=74qn0eJSjpA',
1186 'info_dict': {
1187 'id': '74qn0eJSjpA',
1188 'ext': 'mp4',
1189 'title': 'Eastside',
1190 'description': 'md5:290516bb73dcbfab0dcc4efe6c3de5f2',
1191 'upload_date': '20180710',
1192 'uploader': 'Benny Blanco - Topic',
1193 'uploader_id': 'UCzqz_ksRu_WkIzmivMdIS7A',
1194 'artist': 'benny blanco, Halsey, Khalid',
1195 'track': 'Eastside',
1196 'album': 'Eastside',
1197 'release_date': '20180713',
1198 'release_year': 2018,
1199 },
1200 'params': {
1201 'skip_download': True,
1202 },
1203 },
1204 {
1205 # Youtube Music Auto-generated description
1206 # handle youtube music video with release_year and no release_date
1207 'url': 'https://www.youtube.com/watch?v=-hcAI0g-f5M',
1208 'info_dict': {
1209 'id': '-hcAI0g-f5M',
1210 'ext': 'mp4',
1211 'title': 'Put It On Me',
1212 'description': 'md5:f6422397c07c4c907c6638e1fee380a5',
1213 'upload_date': '20180426',
1214 'uploader': 'Matt Maeson - Topic',
1215 'uploader_id': 'UCnEkIGqtGcQMLk73Kp-Q5LQ',
1216 'artist': 'Matt Maeson',
1217 'track': 'Put It On Me',
1218 'album': 'The Hearse',
1219 'release_date': None,
1220 'release_year': 2018,
1221 },
1222 'params': {
1223 'skip_download': True,
1224 },
1225 },
1226 {
1227 'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1228 'only_matching': True,
1229 },
1230 ]
1231
1232 def __init__(self, *args, **kwargs):
1233 super(YoutubeIE, self).__init__(*args, **kwargs)
1234 self._player_cache = {}
1235
1236 def report_video_info_webpage_download(self, video_id):
1237 """Report attempt to download video info webpage."""
1238 self.to_screen('%s: Downloading video info webpage' % video_id)
1239
1240 def report_information_extraction(self, video_id):
1241 """Report attempt to extract video information."""
1242 self.to_screen('%s: Extracting video information' % video_id)
1243
1244 def report_unavailable_format(self, video_id, format):
1245 """Report extracted video URL."""
1246 self.to_screen('%s: Format %s not available' % (video_id, format))
1247
1248 def report_rtmp_download(self):
1249 """Indicate the download will use the RTMP protocol."""
1250 self.to_screen('RTMP download detected')
1251
1252 def _signature_cache_id(self, example_sig):
1253 """ Return a string representation of a signature """
1254 return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
1255
1256 def _extract_signature_function(self, video_id, player_url, example_sig):
1257 id_m = re.match(
1258 r'.*?[-.](?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
1259 player_url)
1260 if not id_m:
1261 raise ExtractorError('Cannot identify player %r' % player_url)
1262 player_type = id_m.group('ext')
1263 player_id = id_m.group('id')
1264
1265 # Read from filesystem cache
1266 func_id = '%s_%s_%s' % (
1267 player_type, player_id, self._signature_cache_id(example_sig))
1268 assert os.path.basename(func_id) == func_id
1269
1270 cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
1271 if cache_spec is not None:
1272 return lambda s: ''.join(s[i] for i in cache_spec)
1273
1274 download_note = (
1275 'Downloading player %s' % player_url
1276 if self._downloader.params.get('verbose') else
1277 'Downloading %s player %s' % (player_type, player_id)
1278 )
1279 if player_type == 'js':
1280 code = self._download_webpage(
1281 player_url, video_id,
1282 note=download_note,
1283 errnote='Download of %s failed' % player_url)
1284 res = self._parse_sig_js(code)
1285 elif player_type == 'swf':
1286 urlh = self._request_webpage(
1287 player_url, video_id,
1288 note=download_note,
1289 errnote='Download of %s failed' % player_url)
1290 code = urlh.read()
1291 res = self._parse_sig_swf(code)
1292 else:
1293 assert False, 'Invalid player type %r' % player_type
1294
1295 test_string = ''.join(map(compat_chr, range(len(example_sig))))
1296 cache_res = res(test_string)
1297 cache_spec = [ord(c) for c in cache_res]
1298
1299 self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
1300 return res
1301
1302 def _print_sig_code(self, func, example_sig):
1303 def gen_sig_code(idxs):
1304 def _genslice(start, end, step):
1305 starts = '' if start == 0 else str(start)
1306 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
1307 steps = '' if step == 1 else (':%d' % step)
1308 return 's[%s%s%s]' % (starts, ends, steps)
1309
1310 step = None
1311 # Quelch pyflakes warnings - start will be set when step is set
1312 start = '(Never used)'
1313 for i, prev in zip(idxs[1:], idxs[:-1]):
1314 if step is not None:
1315 if i - prev == step:
1316 continue
1317 yield _genslice(start, prev, step)
1318 step = None
1319 continue
1320 if i - prev in [-1, 1]:
1321 step = i - prev
1322 start = prev
1323 continue
1324 else:
1325 yield 's[%d]' % prev
1326 if step is None:
1327 yield 's[%d]' % i
1328 else:
1329 yield _genslice(start, i, step)
1330
1331 test_string = ''.join(map(compat_chr, range(len(example_sig))))
1332 cache_res = func(test_string)
1333 cache_spec = [ord(c) for c in cache_res]
1334 expr_code = ' + '.join(gen_sig_code(cache_spec))
1335 signature_id_tuple = '(%s)' % (
1336 ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1337 code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1338 ' return %s\n') % (signature_id_tuple, expr_code)
1339 self.to_screen('Extracted signature function:\n' + code)
1340
1341 def _parse_sig_js(self, jscode):
1342 funcname = self._search_regex(
1343 (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1344 r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1345 r'\b(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1346 r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1347 # Obsolete patterns
1348 r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1349 r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
1350 r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1351 r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1352 r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1353 r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1354 r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1355 r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
1356 jscode, 'Initial JS player signature function name', group='sig')
1357
1358 jsi = JSInterpreter(jscode)
1359 initial_function = jsi.extract_function(funcname)
1360 return lambda s: initial_function([s])
1361
1362 def _parse_sig_swf(self, file_contents):
1363 swfi = SWFInterpreter(file_contents)
1364 TARGET_CLASSNAME = 'SignatureDecipher'
1365 searched_class = swfi.extract_class(TARGET_CLASSNAME)
1366 initial_function = swfi.extract_function(searched_class, 'decipher')
1367 return lambda s: initial_function([s])
1368
1369 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1370 """Turn the encrypted s field into a working signature"""
1371
1372 if player_url is None:
1373 raise ExtractorError('Cannot decrypt signature without player_url')
1374
1375 if player_url.startswith('//'):
1376 player_url = 'https:' + player_url
1377 elif not re.match(r'https?://', player_url):
1378 player_url = compat_urlparse.urljoin(
1379 'https://www.youtube.com', player_url)
1380 try:
1381 player_id = (player_url, self._signature_cache_id(s))
1382 if player_id not in self._player_cache:
1383 func = self._extract_signature_function(
1384 video_id, player_url, s
1385 )
1386 self._player_cache[player_id] = func
1387 func = self._player_cache[player_id]
1388 if self._downloader.params.get('youtube_print_sig_code'):
1389 self._print_sig_code(func, s)
1390 return func(s)
1391 except Exception as e:
1392 tb = traceback.format_exc()
1393 raise ExtractorError(
1394 'Signature extraction failed: ' + tb, cause=e)
1395
1396 def _get_subtitles(self, video_id, webpage):
1397 try:
1398 subs_doc = self._download_xml(
1399 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1400 video_id, note=False)
1401 except ExtractorError as err:
1402 self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
1403 return {}
1404
1405 sub_lang_list = {}
1406 for track in subs_doc.findall('track'):
1407 lang = track.attrib['lang_code']
1408 if lang in sub_lang_list:
1409 continue
1410 sub_formats = []
1411 for ext in self._SUBTITLE_FORMATS:
1412 params = compat_urllib_parse_urlencode({
1413 'lang': lang,
1414 'v': video_id,
1415 'fmt': ext,
1416 'name': track.attrib['name'].encode('utf-8'),
1417 })
1418 sub_formats.append({
1419 'url': 'https://www.youtube.com/api/timedtext?' + params,
1420 'ext': ext,
1421 })
1422 sub_lang_list[lang] = sub_formats
1423 if not sub_lang_list:
1424 self._downloader.report_warning('video doesn\'t have subtitles')
1425 return {}
1426 return sub_lang_list
1427
1428 def _get_ytplayer_config(self, video_id, webpage):
1429 patterns = (
1430 # User data may contain arbitrary character sequences that may affect
1431 # JSON extraction with regex, e.g. when '};' is contained the second
1432 # regex won't capture the whole JSON. Yet working around by trying more
1433 # concrete regex first keeping in mind proper quoted string handling
1434 # to be implemented in future that will replace this workaround (see
1435 # https://github.com/ytdl-org/youtube-dl/issues/7468,
1436 # https://github.com/ytdl-org/youtube-dl/pull/7599)
1437 r';ytplayer\.config\s*=\s*({.+?});ytplayer',
1438 r';ytplayer\.config\s*=\s*({.+?});',
1439 )
1440 config = self._search_regex(
1441 patterns, webpage, 'ytplayer.config', default=None)
1442 if config:
1443 return self._parse_json(
1444 uppercase_escape(config), video_id, fatal=False)
1445
1446 def _get_automatic_captions(self, video_id, webpage):
1447 """We need the webpage for getting the captions url, pass it as an
1448 argument to speed up the process."""
1449 self.to_screen('%s: Looking for automatic captions' % video_id)
1450 player_config = self._get_ytplayer_config(video_id, webpage)
1451 err_msg = 'Couldn\'t find automatic captions for %s' % video_id
1452 if not player_config:
1453 self._downloader.report_warning(err_msg)
1454 return {}
1455 try:
1456 args = player_config['args']
1457 caption_url = args.get('ttsurl')
1458 if caption_url:
1459 timestamp = args['timestamp']
1460 # We get the available subtitles
1461 list_params = compat_urllib_parse_urlencode({
1462 'type': 'list',
1463 'tlangs': 1,
1464 'asrs': 1,
1465 })
1466 list_url = caption_url + '&' + list_params
1467 caption_list = self._download_xml(list_url, video_id)
1468 original_lang_node = caption_list.find('track')
1469 if original_lang_node is None:
1470 self._downloader.report_warning('Video doesn\'t have automatic captions')
1471 return {}
1472 original_lang = original_lang_node.attrib['lang_code']
1473 caption_kind = original_lang_node.attrib.get('kind', '')
1474
1475 sub_lang_list = {}
1476 for lang_node in caption_list.findall('target'):
1477 sub_lang = lang_node.attrib['lang_code']
1478 sub_formats = []
1479 for ext in self._SUBTITLE_FORMATS:
1480 params = compat_urllib_parse_urlencode({
1481 'lang': original_lang,
1482 'tlang': sub_lang,
1483 'fmt': ext,
1484 'ts': timestamp,
1485 'kind': caption_kind,
1486 })
1487 sub_formats.append({
1488 'url': caption_url + '&' + params,
1489 'ext': ext,
1490 })
1491 sub_lang_list[sub_lang] = sub_formats
1492 return sub_lang_list
1493
1494 def make_captions(sub_url, sub_langs):
1495 parsed_sub_url = compat_urllib_parse_urlparse(sub_url)
1496 caption_qs = compat_parse_qs(parsed_sub_url.query)
1497 captions = {}
1498 for sub_lang in sub_langs:
1499 sub_formats = []
1500 for ext in self._SUBTITLE_FORMATS:
1501 caption_qs.update({
1502 'tlang': [sub_lang],
1503 'fmt': [ext],
1504 })
1505 sub_url = compat_urlparse.urlunparse(parsed_sub_url._replace(
1506 query=compat_urllib_parse_urlencode(caption_qs, True)))
1507 sub_formats.append({
1508 'url': sub_url,
1509 'ext': ext,
1510 })
1511 captions[sub_lang] = sub_formats
1512 return captions
1513
1514 # New captions format as of 22.06.2017
1515 player_response = args.get('player_response')
1516 if player_response and isinstance(player_response, compat_str):
1517 player_response = self._parse_json(
1518 player_response, video_id, fatal=False)
1519 if player_response:
1520 renderer = player_response['captions']['playerCaptionsTracklistRenderer']
1521 base_url = renderer['captionTracks'][0]['baseUrl']
1522 sub_lang_list = []
1523 for lang in renderer['translationLanguages']:
1524 lang_code = lang.get('languageCode')
1525 if lang_code:
1526 sub_lang_list.append(lang_code)
1527 return make_captions(base_url, sub_lang_list)
1528
1529 # Some videos don't provide ttsurl but rather caption_tracks and
1530 # caption_translation_languages (e.g. 20LmZk1hakA)
1531 # Does not used anymore as of 22.06.2017
1532 caption_tracks = args['caption_tracks']
1533 caption_translation_languages = args['caption_translation_languages']
1534 caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
1535 sub_lang_list = []
1536 for lang in caption_translation_languages.split(','):
1537 lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
1538 sub_lang = lang_qs.get('lc', [None])[0]
1539 if sub_lang:
1540 sub_lang_list.append(sub_lang)
1541 return make_captions(caption_url, sub_lang_list)
1542 # An extractor error can be raise by the download process if there are
1543 # no automatic captions but there are subtitles
1544 except (KeyError, IndexError, ExtractorError):
1545 self._downloader.report_warning(err_msg)
1546 return {}
1547
1548 def _mark_watched(self, video_id, video_info, player_response):
1549 playback_url = url_or_none(try_get(
1550 player_response,
1551 lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']) or try_get(
1552 video_info, lambda x: x['videostats_playback_base_url'][0]))
1553 if not playback_url:
1554 return
1555 parsed_playback_url = compat_urlparse.urlparse(playback_url)
1556 qs = compat_urlparse.parse_qs(parsed_playback_url.query)
1557
1558 # cpn generation algorithm is reverse engineered from base.js.
1559 # In fact it works even with dummy cpn.
1560 CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1561 cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
1562
1563 qs.update({
1564 'ver': ['2'],
1565 'cpn': [cpn],
1566 })
1567 playback_url = compat_urlparse.urlunparse(
1568 parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
1569
1570 self._download_webpage(
1571 playback_url, video_id, 'Marking watched',
1572 'Unable to mark watched', fatal=False)
1573
1574 @staticmethod
1575 def _extract_urls(webpage):
1576 # Embedded YouTube player
1577 entries = [
1578 unescapeHTML(mobj.group('url'))
1579 for mobj in re.finditer(r'''(?x)
1580 (?:
1581 <iframe[^>]+?src=|
1582 data-video-url=|
1583 <embed[^>]+?src=|
1584 embedSWF\(?:\s*|
1585 <object[^>]+data=|
1586 new\s+SWFObject\(
1587 )
1588 (["\'])
1589 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1590 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1591 \1''', webpage)]
1592
1593 # lazyYT YouTube embed
1594 entries.extend(list(map(
1595 unescapeHTML,
1596 re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
1597
1598 # Wordpress "YouTube Video Importer" plugin
1599 matches = re.findall(r'''(?x)<div[^>]+
1600 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1601 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1602 entries.extend(m[-1] for m in matches)
1603
1604 return entries
1605
1606 @staticmethod
1607 def _extract_url(webpage):
1608 urls = YoutubeIE._extract_urls(webpage)
1609 return urls[0] if urls else None
1610
1611 @classmethod
1612 def extract_id(cls, url):
1613 mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1614 if mobj is None:
1615 raise ExtractorError('Invalid URL: %s' % url)
1616 video_id = mobj.group(2)
1617 return video_id
1618
1619 @staticmethod
1620 def _extract_chapters(description, duration):
1621 if not description:
1622 return None
1623 chapter_lines = re.findall(
1624 r'(?:^|<br\s*/>)([^<]*<a[^>]+onclick=["\']yt\.www\.watch\.player\.seekTo[^>]+>(\d{1,2}:\d{1,2}(?::\d{1,2})?)</a>[^>]*)(?=$|<br\s*/>)',
1625 description)
1626 if not chapter_lines:
1627 return None
1628 chapters = []
1629 for next_num, (chapter_line, time_point) in enumerate(
1630 chapter_lines, start=1):
1631 start_time = parse_duration(time_point)
1632 if start_time is None:
1633 continue
1634 if start_time > duration:
1635 break
1636 end_time = (duration if next_num == len(chapter_lines)
1637 else parse_duration(chapter_lines[next_num][1]))
1638 if end_time is None:
1639 continue
1640 if end_time > duration:
1641 end_time = duration
1642 if start_time > end_time:
1643 break
1644 chapter_title = re.sub(
1645 r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
1646 chapter_title = re.sub(r'\s+', ' ', chapter_title)
1647 chapters.append({
1648 'start_time': start_time,
1649 'end_time': end_time,
1650 'title': chapter_title,
1651 })
1652 return chapters
1653
1654 def _real_extract(self, url):
1655 url, smuggled_data = unsmuggle_url(url, {})
1656
1657 proto = (
1658 'http' if self._downloader.params.get('prefer_insecure', False)
1659 else 'https')
1660
1661 start_time = None
1662 end_time = None
1663 parsed_url = compat_urllib_parse_urlparse(url)
1664 for component in [parsed_url.fragment, parsed_url.query]:
1665 query = compat_parse_qs(component)
1666 if start_time is None and 't' in query:
1667 start_time = parse_duration(query['t'][0])
1668 if start_time is None and 'start' in query:
1669 start_time = parse_duration(query['start'][0])
1670 if end_time is None and 'end' in query:
1671 end_time = parse_duration(query['end'][0])
1672
1673 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1674 mobj = re.search(self._NEXT_URL_RE, url)
1675 if mobj:
1676 url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
1677 video_id = self.extract_id(url)
1678
1679 # Get video webpage
1680 url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1681 video_webpage = self._download_webpage(url, video_id)
1682
1683 # Attempt to extract SWF player URL
1684 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1685 if mobj is not None:
1686 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1687 else:
1688 player_url = None
1689
1690 dash_mpds = []
1691
1692 def add_dash_mpd(video_info):
1693 dash_mpd = video_info.get('dashmpd')
1694 if dash_mpd and dash_mpd[0] not in dash_mpds:
1695 dash_mpds.append(dash_mpd[0])
1696
1697 def add_dash_mpd_pr(pl_response):
1698 dash_mpd = url_or_none(try_get(
1699 pl_response, lambda x: x['streamingData']['dashManifestUrl'],
1700 compat_str))
1701 if dash_mpd and dash_mpd not in dash_mpds:
1702 dash_mpds.append(dash_mpd)
1703
1704 is_live = None
1705 view_count = None
1706
1707 def extract_view_count(v_info):
1708 return int_or_none(try_get(v_info, lambda x: x['view_count'][0]))
1709
1710 def extract_player_response(player_response, video_id):
1711 pl_response = str_or_none(player_response)
1712 if not pl_response:
1713 return
1714 pl_response = self._parse_json(pl_response, video_id, fatal=False)
1715 if isinstance(pl_response, dict):
1716 add_dash_mpd_pr(pl_response)
1717 return pl_response
1718
1719 player_response = {}
1720
1721 # Get video info
1722 video_info = {}
1723 embed_webpage = None
1724 if re.search(r'player-age-gate-content">', video_webpage) is not None:
1725 age_gate = True
1726 # We simulate the access to the video from www.youtube.com/v/{video_id}
1727 # this can be viewed without login into Youtube
1728 url = proto + '://www.youtube.com/embed/%s' % video_id
1729 embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
1730 data = compat_urllib_parse_urlencode({
1731 'video_id': video_id,
1732 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1733 'sts': self._search_regex(
1734 r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
1735 })
1736 video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1737 try:
1738 video_info_webpage = self._download_webpage(
1739 video_info_url, video_id,
1740 note='Refetching age-gated info webpage',
1741 errnote='unable to download video info webpage')
1742 except ExtractorError:
1743 video_info_webpage = None
1744 if video_info_webpage:
1745 video_info = compat_parse_qs(video_info_webpage)
1746 pl_response = video_info.get('player_response', [None])[0]
1747 player_response = extract_player_response(pl_response, video_id)
1748 add_dash_mpd(video_info)
1749 view_count = extract_view_count(video_info)
1750 else:
1751 age_gate = False
1752 # Try looking directly into the video webpage
1753 ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
1754 if ytplayer_config:
1755 args = ytplayer_config['args']
1756 if args.get('url_encoded_fmt_stream_map') or args.get('hlsvp'):
1757 # Convert to the same format returned by compat_parse_qs
1758 video_info = dict((k, [v]) for k, v in args.items())
1759 add_dash_mpd(video_info)
1760 # Rental video is not rented but preview is available (e.g.
1761 # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
1762 # https://github.com/ytdl-org/youtube-dl/issues/10532)
1763 if not video_info and args.get('ypc_vid'):
1764 return self.url_result(
1765 args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
1766 if args.get('livestream') == '1' or args.get('live_playback') == 1:
1767 is_live = True
1768 if not player_response:
1769 player_response = extract_player_response(args.get('player_response'), video_id)
1770 if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
1771 add_dash_mpd_pr(player_response)
1772
1773 def extract_unavailable_message():
1774 messages = []
1775 for tag, kind in (('h1', 'message'), ('div', 'submessage')):
1776 msg = self._html_search_regex(
1777 r'(?s)<{tag}[^>]+id=["\']unavailable-{kind}["\'][^>]*>(.+?)</{tag}>'.format(tag=tag, kind=kind),
1778 video_webpage, 'unavailable %s' % kind, default=None)
1779 if msg:
1780 messages.append(msg)
1781 if messages:
1782 return '\n'.join(messages)
1783
1784 if not video_info and not player_response:
1785 unavailable_message = extract_unavailable_message()
1786 if not unavailable_message:
1787 unavailable_message = 'Unable to extract video data'
1788 raise ExtractorError(
1789 'YouTube said: %s' % unavailable_message, expected=True, video_id=video_id)
1790
1791 if not isinstance(video_info, dict):
1792 video_info = {}
1793
1794 video_details = try_get(
1795 player_response, lambda x: x['videoDetails'], dict) or {}
1796
1797 video_title = video_info.get('title', [None])[0] or video_details.get('title')
1798 if not video_title:
1799 self._downloader.report_warning('Unable to extract video title')
1800 video_title = '_'
1801
1802 description_original = video_description = get_element_by_id("eow-description", video_webpage)
1803 if video_description:
1804
1805 def replace_url(m):
1806 redir_url = compat_urlparse.urljoin(url, m.group(1))
1807 parsed_redir_url = compat_urllib_parse_urlparse(redir_url)
1808 if re.search(r'^(?:www\.)?(?:youtube(?:-nocookie)?\.com|youtu\.be)$', parsed_redir_url.netloc) and parsed_redir_url.path == '/redirect':
1809 qs = compat_parse_qs(parsed_redir_url.query)
1810 q = qs.get('q')
1811 if q and q[0]:
1812 return q[0]
1813 return redir_url
1814
1815 description_original = video_description = re.sub(r'''(?x)
1816 <a\s+
1817 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1818 (?:title|href)="([^"]+)"\s+
1819 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1820 class="[^"]*"[^>]*>
1821 [^<]+\.{3}\s*
1822 </a>
1823 ''', replace_url, video_description)
1824 video_description = clean_html(video_description)
1825 else:
1826 video_description = self._html_search_meta('description', video_webpage) or video_details.get('shortDescription')
1827
1828 if not smuggled_data.get('force_singlefeed', False):
1829 if not self._downloader.params.get('noplaylist'):
1830 multifeed_metadata_list = try_get(
1831 player_response,
1832 lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
1833 compat_str) or try_get(
1834 video_info, lambda x: x['multifeed_metadata_list'][0], compat_str)
1835 if multifeed_metadata_list:
1836 entries = []
1837 feed_ids = []
1838 for feed in multifeed_metadata_list.split(','):
1839 # Unquote should take place before split on comma (,) since textual
1840 # fields may contain comma as well (see
1841 # https://github.com/ytdl-org/youtube-dl/issues/8536)
1842 feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
1843 entries.append({
1844 '_type': 'url_transparent',
1845 'ie_key': 'Youtube',
1846 'url': smuggle_url(
1847 '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
1848 {'force_singlefeed': True}),
1849 'title': '%s (%s)' % (video_title, feed_data['title'][0]),
1850 })
1851 feed_ids.append(feed_data['id'][0])
1852 self.to_screen(
1853 'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1854 % (', '.join(feed_ids), video_id))
1855 return self.playlist_result(entries, video_id, video_title, video_description)
1856 else:
1857 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1858
1859 if view_count is None:
1860 view_count = extract_view_count(video_info)
1861 if view_count is None and video_details:
1862 view_count = int_or_none(video_details.get('viewCount'))
1863
1864 if is_live is None:
1865 is_live = bool_or_none(video_details.get('isLive'))
1866
1867 # Check for "rental" videos
1868 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1869 raise ExtractorError('"rental" videos not supported. See https://github.com/ytdl-org/youtube-dl/issues/359 for more information.', expected=True)
1870
1871 def _extract_filesize(media_url):
1872 return int_or_none(self._search_regex(
1873 r'\bclen[=/](\d+)', media_url, 'filesize', default=None))
1874
1875 streaming_formats = try_get(player_response, lambda x: x['streamingData']['formats'], list) or []
1876 streaming_formats.extend(try_get(player_response, lambda x: x['streamingData']['adaptiveFormats'], list) or [])
1877
1878 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1879 self.report_rtmp_download()
1880 formats = [{
1881 'format_id': '_rtmp',
1882 'protocol': 'rtmp',
1883 'url': video_info['conn'][0],
1884 'player_url': player_url,
1885 }]
1886 elif not is_live and (streaming_formats or len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1):
1887 encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
1888 if 'rtmpe%3Dyes' in encoded_url_map:
1889 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/ytdl-org/youtube-dl/issues/343 for more information.', expected=True)
1890 formats = []
1891 formats_spec = {}
1892 fmt_list = video_info.get('fmt_list', [''])[0]
1893 if fmt_list:
1894 for fmt in fmt_list.split(','):
1895 spec = fmt.split('/')
1896 if len(spec) > 1:
1897 width_height = spec[1].split('x')
1898 if len(width_height) == 2:
1899 formats_spec[spec[0]] = {
1900 'resolution': spec[1],
1901 'width': int_or_none(width_height[0]),
1902 'height': int_or_none(width_height[1]),
1903 }
1904 for fmt in streaming_formats:
1905 itag = str_or_none(fmt.get('itag'))
1906 if not itag:
1907 continue
1908 quality = fmt.get('quality')
1909 quality_label = fmt.get('qualityLabel') or quality
1910 formats_spec[itag] = {
1911 'asr': int_or_none(fmt.get('audioSampleRate')),
1912 'filesize': int_or_none(fmt.get('contentLength')),
1913 'format_note': quality_label,
1914 'fps': int_or_none(fmt.get('fps')),
1915 'height': int_or_none(fmt.get('height')),
1916 # bitrate for itag 43 is always 2147483647
1917 'tbr': float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000) if itag != '43' else None,
1918 'width': int_or_none(fmt.get('width')),
1919 }
1920
1921 for fmt in streaming_formats:
1922 if fmt.get('drm_families'):
1923 continue
1924 url = url_or_none(fmt.get('url'))
1925
1926 if not url:
1927 cipher = fmt.get('cipher')
1928 if not cipher:
1929 continue
1930 url_data = compat_parse_qs(cipher)
1931 url = url_or_none(try_get(url_data, lambda x: x['url'][0], compat_str))
1932 if not url:
1933 continue
1934 else:
1935 cipher = None
1936 url_data = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
1937
1938 stream_type = int_or_none(try_get(url_data, lambda x: x['stream_type'][0]))
1939 # Unsupported FORMAT_STREAM_TYPE_OTF
1940 if stream_type == 3:
1941 continue
1942
1943 format_id = fmt.get('itag') or url_data['itag'][0]
1944 if not format_id:
1945 continue
1946 format_id = compat_str(format_id)
1947
1948 if cipher:
1949 if 's' in url_data or self._downloader.params.get('youtube_include_dash_manifest', True):
1950 ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
1951 jsplayer_url_json = self._search_regex(
1952 ASSETS_RE,
1953 embed_webpage if age_gate else video_webpage,
1954 'JS player URL (1)', default=None)
1955 if not jsplayer_url_json and not age_gate:
1956 # We need the embed website after all
1957 if embed_webpage is None:
1958 embed_url = proto + '://www.youtube.com/embed/%s' % video_id
1959 embed_webpage = self._download_webpage(
1960 embed_url, video_id, 'Downloading embed webpage')
1961 jsplayer_url_json = self._search_regex(
1962 ASSETS_RE, embed_webpage, 'JS player URL')
1963
1964 player_url = json.loads(jsplayer_url_json)
1965 if player_url is None:
1966 player_url_json = self._search_regex(
1967 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
1968 video_webpage, 'age gate player URL')
1969 player_url = json.loads(player_url_json)
1970
1971 if 'sig' in url_data:
1972 url += '&signature=' + url_data['sig'][0]
1973 elif 's' in url_data:
1974 encrypted_sig = url_data['s'][0]
1975
1976 if self._downloader.params.get('verbose'):
1977 if player_url is None:
1978 player_version = 'unknown'
1979 player_desc = 'unknown'
1980 else:
1981 if player_url.endswith('swf'):
1982 player_version = self._search_regex(
1983 r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
1984 'flash player', fatal=False)
1985 player_desc = 'flash player %s' % player_version
1986 else:
1987 player_version = self._search_regex(
1988 [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
1989 r'(?:www|player(?:_ias)?)[-.]([^/]+)(?:/[a-z]{2,3}_[A-Z]{2})?/base\.js'],
1990 player_url,
1991 'html5 player', fatal=False)
1992 player_desc = 'html5 player %s' % player_version
1993
1994 parts_sizes = self._signature_cache_id(encrypted_sig)
1995 self.to_screen('{%s} signature length %s, %s' %
1996 (format_id, parts_sizes, player_desc))
1997
1998 signature = self._decrypt_signature(
1999 encrypted_sig, video_id, player_url, age_gate)
2000 sp = try_get(url_data, lambda x: x['sp'][0], compat_str) or 'signature'
2001 url += '&%s=%s' % (sp, signature)
2002 if 'ratebypass' not in url:
2003 url += '&ratebypass=yes'
2004
2005 dct = {
2006 'format_id': format_id,
2007 'url': url,
2008 'player_url': player_url,
2009 }
2010 if format_id in self._formats:
2011 dct.update(self._formats[format_id])
2012 if format_id in formats_spec:
2013 dct.update(formats_spec[format_id])
2014
2015 # Some itags are not included in DASH manifest thus corresponding formats will
2016 # lack metadata (see https://github.com/ytdl-org/youtube-dl/pull/5993).
2017 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
2018 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
2019 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
2020
2021 if width is None:
2022 width = int_or_none(fmt.get('width'))
2023 if height is None:
2024 height = int_or_none(fmt.get('height'))
2025
2026 filesize = int_or_none(url_data.get(
2027 'clen', [None])[0]) or _extract_filesize(url)
2028
2029 quality = url_data.get('quality', [None])[0] or fmt.get('quality')
2030 quality_label = url_data.get('quality_label', [None])[0] or fmt.get('qualityLabel')
2031
2032 tbr = (float_or_none(url_data.get('bitrate', [None])[0], 1000)
2033 or float_or_none(fmt.get('bitrate'), 1000)) if format_id != '43' else None
2034 fps = int_or_none(url_data.get('fps', [None])[0]) or int_or_none(fmt.get('fps'))
2035
2036 more_fields = {
2037 'filesize': filesize,
2038 'tbr': tbr,
2039 'width': width,
2040 'height': height,
2041 'fps': fps,
2042 'format_note': quality_label or quality,
2043 }
2044 for key, value in more_fields.items():
2045 if value:
2046 dct[key] = value
2047 type_ = url_data.get('type', [None])[0] or fmt.get('mimeType')
2048 if type_:
2049 type_split = type_.split(';')
2050 kind_ext = type_split[0].split('/')
2051 if len(kind_ext) == 2:
2052 kind, _ = kind_ext
2053 dct['ext'] = mimetype2ext(type_split[0])
2054 if kind in ('audio', 'video'):
2055 codecs = None
2056 for mobj in re.finditer(
2057 r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
2058 if mobj.group('key') == 'codecs':
2059 codecs = mobj.group('val')
2060 break
2061 if codecs:
2062 dct.update(parse_codecs(codecs))
2063 if dct.get('acodec') == 'none' or dct.get('vcodec') == 'none':
2064 dct['downloader_options'] = {
2065 # Youtube throttles chunks >~10M
2066 'http_chunk_size': 10485760,
2067 }
2068 formats.append(dct)
2069 else:
2070 manifest_url = (
2071 url_or_none(try_get(
2072 player_response,
2073 lambda x: x['streamingData']['hlsManifestUrl'],
2074 compat_str))
2075 or url_or_none(try_get(
2076 video_info, lambda x: x['hlsvp'][0], compat_str)))
2077 if manifest_url:
2078 formats = []
2079 m3u8_formats = self._extract_m3u8_formats(
2080 manifest_url, video_id, 'mp4', fatal=False)
2081 for a_format in m3u8_formats:
2082 itag = self._search_regex(
2083 r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
2084 if itag:
2085 a_format['format_id'] = itag
2086 if itag in self._formats:
2087 dct = self._formats[itag].copy()
2088 dct.update(a_format)
2089 a_format = dct
2090 a_format['player_url'] = player_url
2091 # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
2092 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
2093 formats.append(a_format)
2094 else:
2095 error_message = extract_unavailable_message()
2096 if not error_message:
2097 error_message = clean_html(try_get(
2098 player_response, lambda x: x['playabilityStatus']['reason'],
2099 compat_str))
2100 if not error_message:
2101 error_message = clean_html(
2102 try_get(video_info, lambda x: x['reason'][0], compat_str))
2103 if error_message:
2104 raise ExtractorError(error_message, expected=True)
2105 raise ExtractorError('no conn, hlsvp, hlsManifestUrl or url_encoded_fmt_stream_map information found in video info')
2106
2107 # uploader
2108 video_uploader = try_get(
2109 video_info, lambda x: x['author'][0],
2110 compat_str) or str_or_none(video_details.get('author'))
2111 if video_uploader:
2112 video_uploader = compat_urllib_parse_unquote_plus(video_uploader)
2113 else:
2114 self._downloader.report_warning('unable to extract uploader name')
2115
2116 # uploader_id
2117 video_uploader_id = None
2118 video_uploader_url = None
2119 mobj = re.search(
2120 r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
2121 video_webpage)
2122 if mobj is not None:
2123 video_uploader_id = mobj.group('uploader_id')
2124 video_uploader_url = mobj.group('uploader_url')
2125 else:
2126 self._downloader.report_warning('unable to extract uploader nickname')
2127
2128 channel_id = (
2129 str_or_none(video_details.get('channelId'))
2130 or self._html_search_meta(
2131 'channelId', video_webpage, 'channel id', default=None)
2132 or self._search_regex(
2133 r'data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
2134 video_webpage, 'channel id', default=None, group='id'))
2135 channel_url = 'http://www.youtube.com/channel/%s' % channel_id if channel_id else None
2136
2137 # thumbnail image
2138 # We try first to get a high quality image:
2139 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
2140 video_webpage, re.DOTALL)
2141 if m_thumb is not None:
2142 video_thumbnail = m_thumb.group(1)
2143 elif 'thumbnail_url' not in video_info:
2144 self._downloader.report_warning('unable to extract video thumbnail')
2145 video_thumbnail = None
2146 else: # don't panic if we can't find it
2147 video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
2148
2149 # upload date
2150 upload_date = self._html_search_meta(
2151 'datePublished', video_webpage, 'upload date', default=None)
2152 if not upload_date:
2153 upload_date = self._search_regex(
2154 [r'(?s)id="eow-date.*?>(.*?)</span>',
2155 r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']'],
2156 video_webpage, 'upload date', default=None)
2157 upload_date = unified_strdate(upload_date)
2158
2159 video_license = self._html_search_regex(
2160 r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
2161 video_webpage, 'license', default=None)
2162
2163 m_music = re.search(
2164 r'''(?x)
2165 <h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*
2166 <ul[^>]*>\s*
2167 <li>(?P<title>.+?)
2168 by (?P<creator>.+?)
2169 (?:
2170 \(.+?\)|
2171 <a[^>]*
2172 (?:
2173 \bhref=["\']/red[^>]*>| # drop possible
2174 >\s*Listen ad-free with YouTube Red # YouTube Red ad
2175 )
2176 .*?
2177 )?</li
2178 ''',
2179 video_webpage)
2180 if m_music:
2181 video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
2182 video_creator = clean_html(m_music.group('creator'))
2183 else:
2184 video_alt_title = video_creator = None
2185
2186 def extract_meta(field):
2187 return self._html_search_regex(
2188 r'<h4[^>]+class="title"[^>]*>\s*%s\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li>\s*' % field,
2189 video_webpage, field, default=None)
2190
2191 track = extract_meta('Song')
2192 artist = extract_meta('Artist')
2193 album = extract_meta('Album')
2194
2195 # Youtube Music Auto-generated description
2196 release_date = release_year = None
2197 if video_description:
2198 mobj = re.search(r'(?s)Provided to YouTube by [^\n]+\n+(?P<track>[^Ā·]+)Ā·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?ā„—\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?', video_description)
2199 if mobj:
2200 if not track:
2201 track = mobj.group('track').strip()
2202 if not artist:
2203 artist = mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('Ā·'))
2204 if not album:
2205 album = mobj.group('album'.strip())
2206 release_year = mobj.group('release_year')
2207 release_date = mobj.group('release_date')
2208 if release_date:
2209 release_date = release_date.replace('-', '')
2210 if not release_year:
2211 release_year = int(release_date[:4])
2212 if release_year:
2213 release_year = int(release_year)
2214
2215 m_episode = re.search(
2216 r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*ā€¢\s*E(?P<episode>\d+)</span>',
2217 video_webpage)
2218 if m_episode:
2219 series = unescapeHTML(m_episode.group('series'))
2220 season_number = int(m_episode.group('season'))
2221 episode_number = int(m_episode.group('episode'))
2222 else:
2223 series = season_number = episode_number = None
2224
2225 m_cat_container = self._search_regex(
2226 r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
2227 video_webpage, 'categories', default=None)
2228 if m_cat_container:
2229 category = self._html_search_regex(
2230 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
2231 default=None)
2232 video_categories = None if category is None else [category]
2233 else:
2234 video_categories = None
2235
2236 video_tags = [
2237 unescapeHTML(m.group('content'))
2238 for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
2239
2240 def _extract_count(count_name):
2241 return str_to_int(self._search_regex(
2242 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
2243 % re.escape(count_name),
2244 video_webpage, count_name, default=None))
2245
2246 like_count = _extract_count('like')
2247 dislike_count = _extract_count('dislike')
2248
2249 if view_count is None:
2250 view_count = str_to_int(self._search_regex(
2251 r'<[^>]+class=["\']watch-view-count[^>]+>\s*([\d,\s]+)', video_webpage,
2252 'view count', default=None))
2253
2254 average_rating = (
2255 float_or_none(video_details.get('averageRating'))
2256 or try_get(video_info, lambda x: float_or_none(x['avg_rating'][0])))
2257
2258 # subtitles
2259 video_subtitles = self.extract_subtitles(video_id, video_webpage)
2260 automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
2261
2262 video_duration = try_get(
2263 video_info, lambda x: int_or_none(x['length_seconds'][0]))
2264 if not video_duration:
2265 video_duration = int_or_none(video_details.get('lengthSeconds'))
2266 if not video_duration:
2267 video_duration = parse_duration(self._html_search_meta(
2268 'duration', video_webpage, 'video duration'))
2269
2270 # annotations
2271 video_annotations = None
2272 if self._downloader.params.get('writeannotations', False):
2273 xsrf_token = self._search_regex(
2274 r'([\'"])XSRF_TOKEN\1\s*:\s*([\'"])(?P<xsrf_token>[A-Za-z0-9+/=]+)\2',
2275 video_webpage, 'xsrf token', group='xsrf_token', fatal=False)
2276 invideo_url = try_get(
2277 player_response, lambda x: x['annotations'][0]['playerAnnotationsUrlsRenderer']['invideoUrl'], compat_str)
2278 if xsrf_token and invideo_url:
2279 xsrf_field_name = self._search_regex(
2280 r'([\'"])XSRF_FIELD_NAME\1\s*:\s*([\'"])(?P<xsrf_field_name>\w+)\2',
2281 video_webpage, 'xsrf field name',
2282 group='xsrf_field_name', default='session_token')
2283 video_annotations = self._download_webpage(
2284 self._proto_relative_url(invideo_url),
2285 video_id, note='Downloading annotations',
2286 errnote='Unable to download video annotations', fatal=False,
2287 data=urlencode_postdata({xsrf_field_name: xsrf_token}))
2288
2289 chapters = self._extract_chapters(description_original, video_duration)
2290
2291 # Look for the DASH manifest
2292 if self._downloader.params.get('youtube_include_dash_manifest', True):
2293 dash_mpd_fatal = True
2294 for mpd_url in dash_mpds:
2295 dash_formats = {}
2296 try:
2297 def decrypt_sig(mobj):
2298 s = mobj.group(1)
2299 dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
2300 return '/signature/%s' % dec_s
2301
2302 mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
2303
2304 for df in self._extract_mpd_formats(
2305 mpd_url, video_id, fatal=dash_mpd_fatal,
2306 formats_dict=self._formats):
2307 if not df.get('filesize'):
2308 df['filesize'] = _extract_filesize(df['url'])
2309 # Do not overwrite DASH format found in some previous DASH manifest
2310 if df['format_id'] not in dash_formats:
2311 dash_formats[df['format_id']] = df
2312 # Additional DASH manifests may end up in HTTP Error 403 therefore
2313 # allow them to fail without bug report message if we already have
2314 # some DASH manifest succeeded. This is temporary workaround to reduce
2315 # burst of bug reports until we figure out the reason and whether it
2316 # can be fixed at all.
2317 dash_mpd_fatal = False
2318 except (ExtractorError, KeyError) as e:
2319 self.report_warning(
2320 'Skipping DASH manifest: %r' % e, video_id)
2321 if dash_formats:
2322 # Remove the formats we found through non-DASH, they
2323 # contain less info and it can be wrong, because we use
2324 # fixed values (for example the resolution). See
2325 # https://github.com/ytdl-org/youtube-dl/issues/5774 for an
2326 # example.
2327 formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
2328 formats.extend(dash_formats.values())
2329
2330 # Check for malformed aspect ratio
2331 stretched_m = re.search(
2332 r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
2333 video_webpage)
2334 if stretched_m:
2335 w = float(stretched_m.group('w'))
2336 h = float(stretched_m.group('h'))
2337 # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
2338 # We will only process correct ratios.
2339 if w > 0 and h > 0:
2340 ratio = w / h
2341 for f in formats:
2342 if f.get('vcodec') != 'none':
2343 f['stretched_ratio'] = ratio
2344
2345 if not formats:
2346 if 'reason' in video_info:
2347 if 'The uploader has not made this video available in your country.' in video_info['reason']:
2348 regions_allowed = self._html_search_meta(
2349 'regionsAllowed', video_webpage, default=None)
2350 countries = regions_allowed.split(',') if regions_allowed else None
2351 self.raise_geo_restricted(
2352 msg=video_info['reason'][0], countries=countries)
2353 reason = video_info['reason'][0]
2354 if 'Invalid parameters' in reason:
2355 unavailable_message = extract_unavailable_message()
2356 if unavailable_message:
2357 reason = unavailable_message
2358 raise ExtractorError(
2359 'YouTube said: %s' % reason,
2360 expected=True, video_id=video_id)
2361 if video_info.get('license_info') or try_get(player_response, lambda x: x['streamingData']['licenseInfos']):
2362 raise ExtractorError('This video is DRM protected.', expected=True)
2363
2364 self._sort_formats(formats)
2365
2366 self.mark_watched(video_id, video_info, player_response)
2367
2368 return {
2369 'id': video_id,
2370 'uploader': video_uploader,
2371 'uploader_id': video_uploader_id,
2372 'uploader_url': video_uploader_url,
2373 'channel_id': channel_id,
2374 'channel_url': channel_url,
2375 'upload_date': upload_date,
2376 'license': video_license,
2377 'creator': video_creator or artist,
2378 'title': video_title,
2379 'alt_title': video_alt_title or track,
2380 'thumbnail': video_thumbnail,
2381 'description': video_description,
2382 'categories': video_categories,
2383 'tags': video_tags,
2384 'subtitles': video_subtitles,
2385 'automatic_captions': automatic_captions,
2386 'duration': video_duration,
2387 'age_limit': 18 if age_gate else 0,
2388 'annotations': video_annotations,
2389 'chapters': chapters,
2390 'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
2391 'view_count': view_count,
2392 'like_count': like_count,
2393 'dislike_count': dislike_count,
2394 'average_rating': average_rating,
2395 'formats': formats,
2396 'is_live': is_live,
2397 'start_time': start_time,
2398 'end_time': end_time,
2399 'series': series,
2400 'season_number': season_number,
2401 'episode_number': episode_number,
2402 'track': track,
2403 'artist': artist,
2404 'album': album,
2405 'release_date': release_date,
2406 'release_year': release_year,
2407 }
2408
2409
2410 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
2411 IE_DESC = 'YouTube.com playlists'
2412 _VALID_URL = r"""(?x)(?:
2413 (?:https?://)?
2414 (?:\w+\.)?
2415 (?:
2416 (?:
2417 youtube(?:kids)?\.com|
2418 invidio\.us
2419 )
2420 /
2421 (?:
2422 (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/(?:videoseries|[0-9A-Za-z_-]{11}))
2423 \? (?:.*?[&;])*? (?:p|a|list)=
2424 | p/
2425 )|
2426 youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
2427 )
2428 (
2429 (?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)?[0-9A-Za-z-_]{10,}
2430 # Top tracks, they can also include dots
2431 |(?:MC)[\w\.]*
2432 )
2433 .*
2434 |
2435 (%(playlist_id)s)
2436 )""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
2437 _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
2438 _VIDEO_RE_TPL = r'href="\s*/watch\?v=%s(?:&amp;(?:[^"]*?index=(?P<index>\d+))?(?:[^>]+>(?P<title>[^<]+))?)?'
2439 _VIDEO_RE = _VIDEO_RE_TPL % r'(?P<id>[0-9A-Za-z_-]{11})'
2440 IE_NAME = 'youtube:playlist'
2441 _TESTS = [{
2442 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2443 'info_dict': {
2444 'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2445 'uploader': 'Sergey M.',
2446 'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2447 'title': 'youtube-dl public playlist',
2448 },
2449 'playlist_count': 1,
2450 }, {
2451 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2452 'info_dict': {
2453 'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2454 'uploader': 'Sergey M.',
2455 'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2456 'title': 'youtube-dl empty playlist',
2457 },
2458 'playlist_count': 0,
2459 }, {
2460 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
2461 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2462 'info_dict': {
2463 'title': '29C3: Not my department',
2464 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2465 'uploader': 'Christiaan008',
2466 'uploader_id': 'ChRiStIaAn008',
2467 },
2468 'playlist_count': 96,
2469 }, {
2470 'note': 'issue #673',
2471 'url': 'PLBB231211A4F62143',
2472 'info_dict': {
2473 'title': '[OLD]Team Fortress 2 (Class-based LP)',
2474 'id': 'PLBB231211A4F62143',
2475 'uploader': 'Wickydoo',
2476 'uploader_id': 'Wickydoo',
2477 },
2478 'playlist_mincount': 26,
2479 }, {
2480 'note': 'Large playlist',
2481 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
2482 'info_dict': {
2483 'title': 'Uploads from Cauchemar',
2484 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
2485 'uploader': 'Cauchemar',
2486 'uploader_id': 'Cauchemar89',
2487 },
2488 'playlist_mincount': 799,
2489 }, {
2490 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2491 'info_dict': {
2492 'title': 'YDL_safe_search',
2493 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2494 },
2495 'playlist_count': 2,
2496 'skip': 'This playlist is private',
2497 }, {
2498 'note': 'embedded',
2499 'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2500 'playlist_count': 4,
2501 'info_dict': {
2502 'title': 'JODA15',
2503 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2504 'uploader': 'milan',
2505 'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
2506 }
2507 }, {
2508 'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2509 'playlist_mincount': 485,
2510 'info_dict': {
2511 'title': '2018 Chinese New Singles (11/6 updated)',
2512 'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2513 'uploader': 'LBK',
2514 'uploader_id': 'sdragonfang',
2515 }
2516 }, {
2517 'note': 'Embedded SWF player',
2518 'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
2519 'playlist_count': 4,
2520 'info_dict': {
2521 'title': 'JODA7',
2522 'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
2523 },
2524 'skip': 'This playlist does not exist',
2525 }, {
2526 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
2527 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
2528 'info_dict': {
2529 'title': 'Uploads from Interstellar Movie',
2530 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
2531 'uploader': 'Interstellar Movie',
2532 'uploader_id': 'InterstellarMovie1',
2533 },
2534 'playlist_mincount': 21,
2535 }, {
2536 # Playlist URL that does not actually serve a playlist
2537 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
2538 'info_dict': {
2539 'id': 'FqZTN594JQw',
2540 'ext': 'webm',
2541 'title': "Smiley's People 01 detective, Adventure Series, Action",
2542 'uploader': 'STREEM',
2543 'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
2544 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
2545 'upload_date': '20150526',
2546 'license': 'Standard YouTube License',
2547 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
2548 'categories': ['People & Blogs'],
2549 'tags': list,
2550 'view_count': int,
2551 'like_count': int,
2552 'dislike_count': int,
2553 },
2554 'params': {
2555 'skip_download': True,
2556 },
2557 'skip': 'This video is not available.',
2558 'add_ie': [YoutubeIE.ie_key()],
2559 }, {
2560 'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
2561 'info_dict': {
2562 'id': 'yeWKywCrFtk',
2563 'ext': 'mp4',
2564 'title': 'Small Scale Baler and Braiding Rugs',
2565 'uploader': 'Backus-Page House Museum',
2566 'uploader_id': 'backuspagemuseum',
2567 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
2568 'upload_date': '20161008',
2569 'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
2570 'categories': ['Nonprofits & Activism'],
2571 'tags': list,
2572 'like_count': int,
2573 'dislike_count': int,
2574 },
2575 'params': {
2576 'noplaylist': True,
2577 'skip_download': True,
2578 },
2579 }, {
2580 # https://github.com/ytdl-org/youtube-dl/issues/21844
2581 'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2582 'info_dict': {
2583 'title': 'Data Analysis with Dr Mike Pound',
2584 'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2585 'uploader_id': 'Computerphile',
2586 'uploader': 'Computerphile',
2587 },
2588 'playlist_mincount': 11,
2589 }, {
2590 'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
2591 'only_matching': True,
2592 }, {
2593 'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
2594 'only_matching': True,
2595 }, {
2596 # music album playlist
2597 'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
2598 'only_matching': True,
2599 }, {
2600 'url': 'https://invidio.us/playlist?list=PLDIoUOhQQPlXr63I_vwF9GD8sAKh77dWU',
2601 'only_matching': True,
2602 }, {
2603 'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
2604 'only_matching': True,
2605 }]
2606
2607 def _real_initialize(self):
2608 self._login()
2609
2610 def extract_videos_from_page(self, page):
2611 ids_in_page = []
2612 titles_in_page = []
2613
2614 for item in re.findall(
2615 r'(<[^>]*\bdata-video-id\s*=\s*["\'][0-9A-Za-z_-]{11}[^>]+>)', page):
2616 attrs = extract_attributes(item)
2617 video_id = attrs['data-video-id']
2618 video_title = unescapeHTML(attrs.get('data-title'))
2619 if video_title:
2620 video_title = video_title.strip()
2621 ids_in_page.append(video_id)
2622 titles_in_page.append(video_title)
2623
2624 # Fallback with old _VIDEO_RE
2625 self.extract_videos_from_page_impl(
2626 self._VIDEO_RE, page, ids_in_page, titles_in_page)
2627
2628 # Relaxed fallbacks
2629 self.extract_videos_from_page_impl(
2630 r'href="\s*/watch\?v\s*=\s*(?P<id>[0-9A-Za-z_-]{11})', page,
2631 ids_in_page, titles_in_page)
2632 self.extract_videos_from_page_impl(
2633 r'data-video-ids\s*=\s*["\'](?P<id>[0-9A-Za-z_-]{11})', page,
2634 ids_in_page, titles_in_page)
2635
2636 return zip(ids_in_page, titles_in_page)
2637
2638 def _extract_mix(self, playlist_id):
2639 # The mixes are generated from a single video
2640 # the id of the playlist is just 'RD' + video_id
2641 ids = []
2642 last_id = playlist_id[-11:]
2643 for n in itertools.count(1):
2644 url = 'https://youtube.com/watch?v=%s&list=%s' % (last_id, playlist_id)
2645 webpage = self._download_webpage(
2646 url, playlist_id, 'Downloading page {0} of Youtube mix'.format(n))
2647 new_ids = orderedSet(re.findall(
2648 r'''(?xs)data-video-username=".*?".*?
2649 href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
2650 webpage))
2651 # Fetch new pages until all the videos are repeated, it seems that
2652 # there are always 51 unique videos.
2653 new_ids = [_id for _id in new_ids if _id not in ids]
2654 if not new_ids:
2655 break
2656 ids.extend(new_ids)
2657 last_id = ids[-1]
2658
2659 url_results = self._ids_to_results(ids)
2660
2661 search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
2662 title_span = (
2663 search_title('playlist-title')
2664 or search_title('title long-title')
2665 or search_title('title'))
2666 title = clean_html(title_span)
2667
2668 return self.playlist_result(url_results, playlist_id, title)
2669
2670 def _extract_playlist(self, playlist_id):
2671 url = self._TEMPLATE_URL % playlist_id
2672 page = self._download_webpage(url, playlist_id)
2673
2674 # the yt-alert-message now has tabindex attribute (see https://github.com/ytdl-org/youtube-dl/issues/11604)
2675 for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
2676 match = match.strip()
2677 # Check if the playlist exists or is private
2678 mobj = re.match(r'[^<]*(?:The|This) playlist (?P<reason>does not exist|is private)[^<]*', match)
2679 if mobj:
2680 reason = mobj.group('reason')
2681 message = 'This playlist %s' % reason
2682 if 'private' in reason:
2683 message += ', use --username or --netrc to access it'
2684 message += '.'
2685 raise ExtractorError(message, expected=True)
2686 elif re.match(r'[^<]*Invalid parameters[^<]*', match):
2687 raise ExtractorError(
2688 'Invalid parameters. Maybe URL is incorrect.',
2689 expected=True)
2690 elif re.match(r'[^<]*Choose your language[^<]*', match):
2691 continue
2692 else:
2693 self.report_warning('Youtube gives an alert message: ' + match)
2694
2695 playlist_title = self._html_search_regex(
2696 r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
2697 page, 'title', default=None)
2698
2699 _UPLOADER_BASE = r'class=["\']pl-header-details[^>]+>\s*<li>\s*<a[^>]+\bhref='
2700 uploader = self._html_search_regex(
2701 r'%s["\']/(?:user|channel)/[^>]+>([^<]+)' % _UPLOADER_BASE,
2702 page, 'uploader', default=None)
2703 mobj = re.search(
2704 r'%s(["\'])(?P<path>/(?:user|channel)/(?P<uploader_id>.+?))\1' % _UPLOADER_BASE,
2705 page)
2706 if mobj:
2707 uploader_id = mobj.group('uploader_id')
2708 uploader_url = compat_urlparse.urljoin(url, mobj.group('path'))
2709 else:
2710 uploader_id = uploader_url = None
2711
2712 has_videos = True
2713
2714 if not playlist_title:
2715 try:
2716 # Some playlist URLs don't actually serve a playlist (e.g.
2717 # https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4)
2718 next(self._entries(page, playlist_id))
2719 except StopIteration:
2720 has_videos = False
2721
2722 playlist = self.playlist_result(
2723 self._entries(page, playlist_id), playlist_id, playlist_title)
2724 playlist.update({
2725 'uploader': uploader,
2726 'uploader_id': uploader_id,
2727 'uploader_url': uploader_url,
2728 })
2729
2730 return has_videos, playlist
2731
2732 def _check_download_just_video(self, url, playlist_id):
2733 # Check if it's a video-specific URL
2734 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
2735 video_id = query_dict.get('v', [None])[0] or self._search_regex(
2736 r'(?:(?:^|//)youtu\.be/|youtube\.com/embed/(?!videoseries))([0-9A-Za-z_-]{11})', url,
2737 'video id', default=None)
2738 if video_id:
2739 if self._downloader.params.get('noplaylist'):
2740 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2741 return video_id, self.url_result(video_id, 'Youtube', video_id=video_id)
2742 else:
2743 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
2744 return video_id, None
2745 return None, None
2746
2747 def _real_extract(self, url):
2748 # Extract playlist id
2749 mobj = re.match(self._VALID_URL, url)
2750 if mobj is None:
2751 raise ExtractorError('Invalid URL: %s' % url)
2752 playlist_id = mobj.group(1) or mobj.group(2)
2753
2754 video_id, video = self._check_download_just_video(url, playlist_id)
2755 if video:
2756 return video
2757
2758 if playlist_id.startswith(('RD', 'UL', 'PU')):
2759 # Mixes require a custom extraction process
2760 return self._extract_mix(playlist_id)
2761
2762 has_videos, playlist = self._extract_playlist(playlist_id)
2763 if has_videos or not video_id:
2764 return playlist
2765
2766 # Some playlist URLs don't actually serve a playlist (see
2767 # https://github.com/ytdl-org/youtube-dl/issues/10537).
2768 # Fallback to plain video extraction if there is a video id
2769 # along with playlist id.
2770 return self.url_result(video_id, 'Youtube', video_id=video_id)
2771
2772
2773 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
2774 IE_DESC = 'YouTube.com channels'
2775 _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie|kids)?\.com|(?:www\.)?invidio\.us)/channel/(?P<id>[0-9A-Za-z_-]+)'
2776 _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
2777 _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
2778 IE_NAME = 'youtube:channel'
2779 _TESTS = [{
2780 'note': 'paginated channel',
2781 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
2782 'playlist_mincount': 91,
2783 'info_dict': {
2784 'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
2785 'title': 'Uploads from lex will',
2786 'uploader': 'lex will',
2787 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2788 }
2789 }, {
2790 'note': 'Age restricted channel',
2791 # from https://www.youtube.com/user/DeusExOfficial
2792 'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
2793 'playlist_mincount': 64,
2794 'info_dict': {
2795 'id': 'UUs0ifCMCm1icqRbqhUINa0w',
2796 'title': 'Uploads from Deus Ex',
2797 'uploader': 'Deus Ex',
2798 'uploader_id': 'DeusExOfficial',
2799 },
2800 }, {
2801 'url': 'https://invidio.us/channel/UC23qupoDRn9YOAVzeoxjOQA',
2802 'only_matching': True,
2803 }, {
2804 'url': 'https://www.youtubekids.com/channel/UCyu8StPfZWapR6rfW_JgqcA',
2805 'only_matching': True,
2806 }]
2807
2808 @classmethod
2809 def suitable(cls, url):
2810 return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
2811 else super(YoutubeChannelIE, cls).suitable(url))
2812
2813 def _build_template_url(self, url, channel_id):
2814 return self._TEMPLATE_URL % channel_id
2815
2816 def _real_extract(self, url):
2817 channel_id = self._match_id(url)
2818
2819 url = self._build_template_url(url, channel_id)
2820
2821 # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
2822 # Workaround by extracting as a playlist if managed to obtain channel playlist URL
2823 # otherwise fallback on channel by page extraction
2824 channel_page = self._download_webpage(
2825 url + '?view=57', channel_id,
2826 'Downloading channel page', fatal=False)
2827 if channel_page is False:
2828 channel_playlist_id = False
2829 else:
2830 channel_playlist_id = self._html_search_meta(
2831 'channelId', channel_page, 'channel id', default=None)
2832 if not channel_playlist_id:
2833 channel_url = self._html_search_meta(
2834 ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
2835 channel_page, 'channel url', default=None)
2836 if channel_url:
2837 channel_playlist_id = self._search_regex(
2838 r'vnd\.youtube://user/([0-9A-Za-z_-]+)',
2839 channel_url, 'channel id', default=None)
2840 if channel_playlist_id and channel_playlist_id.startswith('UC'):
2841 playlist_id = 'UU' + channel_playlist_id[2:]
2842 return self.url_result(
2843 compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
2844
2845 channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
2846 autogenerated = re.search(r'''(?x)
2847 class="[^"]*?(?:
2848 channel-header-autogenerated-label|
2849 yt-channel-title-autogenerated
2850 )[^"]*"''', channel_page) is not None
2851
2852 if autogenerated:
2853 # The videos are contained in a single page
2854 # the ajax pages can't be used, they are empty
2855 entries = [
2856 self.url_result(
2857 video_id, 'Youtube', video_id=video_id,
2858 video_title=video_title)
2859 for video_id, video_title in self.extract_videos_from_page(channel_page)]
2860 return self.playlist_result(entries, channel_id)
2861
2862 try:
2863 next(self._entries(channel_page, channel_id))
2864 except StopIteration:
2865 alert_message = self._html_search_regex(
2866 r'(?s)<div[^>]+class=(["\']).*?\byt-alert-message\b.*?\1[^>]*>(?P<alert>[^<]+)</div>',
2867 channel_page, 'alert', default=None, group='alert')
2868 if alert_message:
2869 raise ExtractorError('Youtube said: %s' % alert_message, expected=True)
2870
2871 return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
2872
2873
2874 class YoutubeUserIE(YoutubeChannelIE):
2875 IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
2876 _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:(?P<user>user|c)/)?(?!(?:attribution_link|watch|results|shared)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
2877 _TEMPLATE_URL = 'https://www.youtube.com/%s/%s/videos'
2878 IE_NAME = 'youtube:user'
2879
2880 _TESTS = [{
2881 'url': 'https://www.youtube.com/user/TheLinuxFoundation',
2882 'playlist_mincount': 320,
2883 'info_dict': {
2884 'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
2885 'title': 'Uploads from The Linux Foundation',
2886 'uploader': 'The Linux Foundation',
2887 'uploader_id': 'TheLinuxFoundation',
2888 }
2889 }, {
2890 # Only available via https://www.youtube.com/c/12minuteathlete/videos
2891 # but not https://www.youtube.com/user/12minuteathlete/videos
2892 'url': 'https://www.youtube.com/c/12minuteathlete/videos',
2893 'playlist_mincount': 249,
2894 'info_dict': {
2895 'id': 'UUVjM-zV6_opMDx7WYxnjZiQ',
2896 'title': 'Uploads from 12 Minute Athlete',
2897 'uploader': '12 Minute Athlete',
2898 'uploader_id': 'the12minuteathlete',
2899 }
2900 }, {
2901 'url': 'ytuser:phihag',
2902 'only_matching': True,
2903 }, {
2904 'url': 'https://www.youtube.com/c/gametrailers',
2905 'only_matching': True,
2906 }, {
2907 'url': 'https://www.youtube.com/gametrailers',
2908 'only_matching': True,
2909 }, {
2910 # This channel is not available, geo restricted to JP
2911 'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
2912 'only_matching': True,
2913 }]
2914
2915 @classmethod
2916 def suitable(cls, url):
2917 # Don't return True if the url can be extracted with other youtube
2918 # extractor, the regex would is too permissive and it would match.
2919 other_yt_ies = iter(klass for (name, klass) in globals().items() if name.startswith('Youtube') and name.endswith('IE') and klass is not cls)
2920 if any(ie.suitable(url) for ie in other_yt_ies):
2921 return False
2922 else:
2923 return super(YoutubeUserIE, cls).suitable(url)
2924
2925 def _build_template_url(self, url, channel_id):
2926 mobj = re.match(self._VALID_URL, url)
2927 return self._TEMPLATE_URL % (mobj.group('user') or 'user', mobj.group('id'))
2928
2929
2930 class YoutubeLiveIE(YoutubeBaseInfoExtractor):
2931 IE_DESC = 'YouTube.com live streams'
2932 _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:(?:user|channel|c)/)?(?P<id>[^/]+))/live'
2933 IE_NAME = 'youtube:live'
2934
2935 _TESTS = [{
2936 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
2937 'info_dict': {
2938 'id': 'a48o2S1cPoo',
2939 'ext': 'mp4',
2940 'title': 'The Young Turks - Live Main Show',
2941 'uploader': 'The Young Turks',
2942 'uploader_id': 'TheYoungTurks',
2943 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
2944 'upload_date': '20150715',
2945 'license': 'Standard YouTube License',
2946 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
2947 'categories': ['News & Politics'],
2948 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
2949 'like_count': int,
2950 'dislike_count': int,
2951 },
2952 'params': {
2953 'skip_download': True,
2954 },
2955 }, {
2956 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
2957 'only_matching': True,
2958 }, {
2959 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
2960 'only_matching': True,
2961 }, {
2962 'url': 'https://www.youtube.com/TheYoungTurks/live',
2963 'only_matching': True,
2964 }]
2965
2966 def _real_extract(self, url):
2967 mobj = re.match(self._VALID_URL, url)
2968 channel_id = mobj.group('id')
2969 base_url = mobj.group('base_url')
2970 webpage = self._download_webpage(url, channel_id, fatal=False)
2971 if webpage:
2972 page_type = self._og_search_property(
2973 'type', webpage, 'page type', default='')
2974 video_id = self._html_search_meta(
2975 'videoId', webpage, 'video id', default=None)
2976 if page_type.startswith('video') and video_id and re.match(
2977 r'^[0-9A-Za-z_-]{11}$', video_id):
2978 return self.url_result(video_id, YoutubeIE.ie_key())
2979 return self.url_result(base_url)
2980
2981
2982 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
2983 IE_DESC = 'YouTube.com user/channel playlists'
2984 _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
2985 IE_NAME = 'youtube:playlists'
2986
2987 _TESTS = [{
2988 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
2989 'playlist_mincount': 4,
2990 'info_dict': {
2991 'id': 'ThirstForScience',
2992 'title': 'ThirstForScience',
2993 },
2994 }, {
2995 # with "Load more" button
2996 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
2997 'playlist_mincount': 70,
2998 'info_dict': {
2999 'id': 'igorkle1',
3000 'title': 'Š˜Š³Š¾Ń€ŃŒ ŠšŠ»ŠµŠ¹Š½ŠµŃ€',
3001 },
3002 }, {
3003 'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
3004 'playlist_mincount': 17,
3005 'info_dict': {
3006 'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
3007 'title': 'Chem Player',
3008 },
3009 'skip': 'Blocked',
3010 }]
3011
3012
3013 class YoutubeSearchBaseInfoExtractor(YoutubePlaylistBaseInfoExtractor):
3014 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
3015
3016
3017 class YoutubeSearchIE(SearchInfoExtractor, YoutubeSearchBaseInfoExtractor):
3018 IE_DESC = 'YouTube.com searches'
3019 # there doesn't appear to be a real limit, for example if you search for
3020 # 'python' you get more than 8.000.000 results
3021 _MAX_RESULTS = float('inf')
3022 IE_NAME = 'youtube:search'
3023 _SEARCH_KEY = 'ytsearch'
3024 _EXTRA_QUERY_ARGS = {}
3025 _TESTS = []
3026
3027 def _get_n_results(self, query, n):
3028 """Get a specified number of results for a query"""
3029
3030 videos = []
3031 limit = n
3032
3033 url_query = {
3034 'search_query': query.encode('utf-8'),
3035 }
3036 url_query.update(self._EXTRA_QUERY_ARGS)
3037 result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
3038
3039 for pagenum in itertools.count(1):
3040 data = self._download_json(
3041 result_url, video_id='query "%s"' % query,
3042 note='Downloading page %s' % pagenum,
3043 errnote='Unable to download API page',
3044 query={'spf': 'navigate'})
3045 html_content = data[1]['body']['content']
3046
3047 if 'class="search-message' in html_content:
3048 raise ExtractorError(
3049 '[youtube] No video results', expected=True)
3050
3051 new_videos = list(self._process_page(html_content))
3052 videos += new_videos
3053 if not new_videos or len(videos) > limit:
3054 break
3055 next_link = self._html_search_regex(
3056 r'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*<span[^>]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next',
3057 html_content, 'next link', default=None)
3058 if next_link is None:
3059 break
3060 result_url = compat_urlparse.urljoin('https://www.youtube.com/', next_link)
3061
3062 if len(videos) > n:
3063 videos = videos[:n]
3064 return self.playlist_result(videos, query)
3065
3066
3067 class YoutubeSearchDateIE(YoutubeSearchIE):
3068 IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
3069 _SEARCH_KEY = 'ytsearchdate'
3070 IE_DESC = 'YouTube.com searches, newest videos first'
3071 _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
3072
3073
3074 class YoutubeSearchURLIE(YoutubeSearchBaseInfoExtractor):
3075 IE_DESC = 'YouTube.com search URLs'
3076 IE_NAME = 'youtube:search_url'
3077 _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
3078 _TESTS = [{
3079 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
3080 'playlist_mincount': 5,
3081 'info_dict': {
3082 'title': 'youtube-dl test video',
3083 }
3084 }, {
3085 'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
3086 'only_matching': True,
3087 }]
3088
3089 def _real_extract(self, url):
3090 mobj = re.match(self._VALID_URL, url)
3091 query = compat_urllib_parse_unquote_plus(mobj.group('query'))
3092 webpage = self._download_webpage(url, query)
3093 return self.playlist_result(self._process_page(webpage), playlist_title=query)
3094
3095
3096 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
3097 IE_DESC = 'YouTube.com (multi-season) shows'
3098 _VALID_URL = r'https?://(?:www\.)?youtube\.com/show/(?P<id>[^?#]*)'
3099 IE_NAME = 'youtube:show'
3100 _TESTS = [{
3101 'url': 'https://www.youtube.com/show/airdisasters',
3102 'playlist_mincount': 5,
3103 'info_dict': {
3104 'id': 'airdisasters',
3105 'title': 'Air Disasters',
3106 }
3107 }]
3108
3109 def _real_extract(self, url):
3110 playlist_id = self._match_id(url)
3111 return super(YoutubeShowIE, self)._real_extract(
3112 'https://www.youtube.com/show/%s/playlists' % playlist_id)
3113
3114
3115 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
3116 """
3117 Base class for feed extractors
3118 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
3119 """
3120 _LOGIN_REQUIRED = True
3121
3122 @property
3123 def IE_NAME(self):
3124 return 'youtube:%s' % self._FEED_NAME
3125
3126 def _real_initialize(self):
3127 self._login()
3128
3129 def _entries(self, page):
3130 # The extraction process is the same as for playlists, but the regex
3131 # for the video ids doesn't contain an index
3132 ids = []
3133 more_widget_html = content_html = page
3134 for page_num in itertools.count(1):
3135 matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
3136
3137 # 'recommended' feed has infinite 'load more' and each new portion spins
3138 # the same videos in (sometimes) slightly different order, so we'll check
3139 # for unicity and break when portion has no new videos
3140 new_ids = list(filter(lambda video_id: video_id not in ids, orderedSet(matches)))
3141 if not new_ids:
3142 break
3143
3144 ids.extend(new_ids)
3145
3146 for entry in self._ids_to_results(new_ids):
3147 yield entry
3148
3149 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
3150 if not mobj:
3151 break
3152
3153 more = self._download_json(
3154 'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
3155 'Downloading page #%s' % page_num,
3156 transform_source=uppercase_escape)
3157 content_html = more['content_html']
3158 more_widget_html = more['load_more_widget_html']
3159
3160 def _real_extract(self, url):
3161 page = self._download_webpage(
3162 'https://www.youtube.com/feed/%s' % self._FEED_NAME,
3163 self._PLAYLIST_TITLE)
3164 return self.playlist_result(
3165 self._entries(page), playlist_title=self._PLAYLIST_TITLE)
3166
3167
3168 class YoutubeWatchLaterIE(YoutubePlaylistIE):
3169 IE_NAME = 'youtube:watchlater'
3170 IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
3171 _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
3172
3173 _TESTS = [{
3174 'url': 'https://www.youtube.com/playlist?list=WL',
3175 'only_matching': True,
3176 }, {
3177 'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
3178 'only_matching': True,
3179 }]
3180
3181 def _real_extract(self, url):
3182 _, video = self._check_download_just_video(url, 'WL')
3183 if video:
3184 return video
3185 _, playlist = self._extract_playlist('WL')
3186 return playlist
3187
3188
3189 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
3190 IE_NAME = 'youtube:favorites'
3191 IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
3192 _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
3193 _LOGIN_REQUIRED = True
3194
3195 def _real_extract(self, url):
3196 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
3197 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
3198 return self.url_result(playlist_id, 'YoutubePlaylist')
3199
3200
3201 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
3202 IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
3203 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/recommended|:ytrec(?:ommended)?'
3204 _FEED_NAME = 'recommended'
3205 _PLAYLIST_TITLE = 'Youtube Recommended videos'
3206
3207
3208 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
3209 IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
3210 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
3211 _FEED_NAME = 'subscriptions'
3212 _PLAYLIST_TITLE = 'Youtube Subscriptions'
3213
3214
3215 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
3216 IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
3217 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
3218 _FEED_NAME = 'history'
3219 _PLAYLIST_TITLE = 'Youtube History'
3220
3221
3222 class YoutubeTruncatedURLIE(InfoExtractor):
3223 IE_NAME = 'youtube:truncated_url'
3224 IE_DESC = False # Do not list
3225 _VALID_URL = r'''(?x)
3226 (?:https?://)?
3227 (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
3228 (?:watch\?(?:
3229 feature=[a-z_]+|
3230 annotation_id=annotation_[^&]+|
3231 x-yt-cl=[0-9]+|
3232 hl=[^&]*|
3233 t=[0-9]+
3234 )?
3235 |
3236 attribution_link\?a=[^&]+
3237 )
3238 $
3239 '''
3240
3241 _TESTS = [{
3242 'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
3243 'only_matching': True,
3244 }, {
3245 'url': 'https://www.youtube.com/watch?',
3246 'only_matching': True,
3247 }, {
3248 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
3249 'only_matching': True,
3250 }, {
3251 'url': 'https://www.youtube.com/watch?feature=foo',
3252 'only_matching': True,
3253 }, {
3254 'url': 'https://www.youtube.com/watch?hl=en-GB',
3255 'only_matching': True,
3256 }, {
3257 'url': 'https://www.youtube.com/watch?t=2372',
3258 'only_matching': True,
3259 }]
3260
3261 def _real_extract(self, url):
3262 raise ExtractorError(
3263 'Did you forget to quote the URL? Remember that & is a meta '
3264 'character in most shells, so you want to put the URL in quotes, '
3265 'like youtube-dl '
3266 '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
3267 ' or simply youtube-dl BaW_jenozKc .',
3268 expected=True)
3269
3270
3271 class YoutubeTruncatedIDIE(InfoExtractor):
3272 IE_NAME = 'youtube:truncated_id'
3273 IE_DESC = False # Do not list
3274 _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
3275
3276 _TESTS = [{
3277 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
3278 'only_matching': True,
3279 }]
3280
3281 def _real_extract(self, url):
3282 video_id = self._match_id(url)
3283 raise ExtractorError(
3284 'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
3285 expected=True)