]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/youtube.py
Update README.md
[youtubedl] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 from __future__ import unicode_literals
4
5
6 import itertools
7 import json
8 import os.path
9 import random
10 import re
11 import time
12 import traceback
13
14 from .common import InfoExtractor, SearchInfoExtractor
15 from ..jsinterp import JSInterpreter
16 from ..swfinterp import SWFInterpreter
17 from ..compat import (
18 compat_chr,
19 compat_HTTPError,
20 compat_kwargs,
21 compat_parse_qs,
22 compat_urllib_parse_unquote,
23 compat_urllib_parse_unquote_plus,
24 compat_urllib_parse_urlencode,
25 compat_urllib_parse_urlparse,
26 compat_urlparse,
27 compat_str,
28 )
29 from ..utils import (
30 bool_or_none,
31 clean_html,
32 error_to_compat_str,
33 extract_attributes,
34 ExtractorError,
35 float_or_none,
36 get_element_by_attribute,
37 get_element_by_id,
38 int_or_none,
39 mimetype2ext,
40 orderedSet,
41 parse_codecs,
42 parse_duration,
43 remove_quotes,
44 remove_start,
45 smuggle_url,
46 str_or_none,
47 str_to_int,
48 try_get,
49 unescapeHTML,
50 unified_strdate,
51 unsmuggle_url,
52 uppercase_escape,
53 url_or_none,
54 urlencode_postdata,
55 )
56
57
58 class YoutubeBaseInfoExtractor(InfoExtractor):
59 """Provide base functions for Youtube extractors"""
60 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
61 _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
62
63 _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
64 _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
65 _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
66
67 _NETRC_MACHINE = 'youtube'
68 # If True it will raise an error if no login info is provided
69 _LOGIN_REQUIRED = False
70
71 _PLAYLIST_ID_RE = r'(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}'
72
73 _YOUTUBE_CLIENT_HEADERS = {
74 'x-youtube-client-name': '1',
75 'x-youtube-client-version': '1.20200609.04.02',
76 }
77
78 def _set_language(self):
79 self._set_cookie(
80 '.youtube.com', 'PREF', 'f1=50000000&f6=8&hl=en',
81 # YouTube sets the expire time to about two months
82 expire_time=time.time() + 2 * 30 * 24 * 3600)
83
84 def _ids_to_results(self, ids):
85 return [
86 self.url_result(vid_id, 'Youtube', video_id=vid_id)
87 for vid_id in ids]
88
89 def _login(self):
90 """
91 Attempt to log in to YouTube.
92 True is returned if successful or skipped.
93 False is returned if login failed.
94
95 If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
96 """
97 username, password = self._get_login_info()
98 # No authentication to be performed
99 if username is None:
100 if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
101 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
102 return True
103
104 login_page = self._download_webpage(
105 self._LOGIN_URL, None,
106 note='Downloading login page',
107 errnote='unable to fetch login page', fatal=False)
108 if login_page is False:
109 return
110
111 login_form = self._hidden_inputs(login_page)
112
113 def req(url, f_req, note, errnote):
114 data = login_form.copy()
115 data.update({
116 'pstMsg': 1,
117 'checkConnection': 'youtube',
118 'checkedDomains': 'youtube',
119 'hl': 'en',
120 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
121 'f.req': json.dumps(f_req),
122 'flowName': 'GlifWebSignIn',
123 'flowEntry': 'ServiceLogin',
124 # TODO: reverse actual botguard identifier generation algo
125 'bgRequest': '["identifier",""]',
126 })
127 return self._download_json(
128 url, None, note=note, errnote=errnote,
129 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
130 fatal=False,
131 data=urlencode_postdata(data), headers={
132 'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
133 'Google-Accounts-XSRF': 1,
134 })
135
136 def warn(message):
137 self._downloader.report_warning(message)
138
139 lookup_req = [
140 username,
141 None, [], None, 'US', None, None, 2, False, True,
142 [
143 None, None,
144 [2, 1, None, 1,
145 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
146 None, [], 4],
147 1, [None, None, []], None, None, None, True
148 ],
149 username,
150 ]
151
152 lookup_results = req(
153 self._LOOKUP_URL, lookup_req,
154 'Looking up account info', 'Unable to look up account info')
155
156 if lookup_results is False:
157 return False
158
159 user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
160 if not user_hash:
161 warn('Unable to extract user hash')
162 return False
163
164 challenge_req = [
165 user_hash,
166 None, 1, None, [1, None, None, None, [password, None, True]],
167 [
168 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
169 1, [None, None, []], None, None, None, True
170 ]]
171
172 challenge_results = req(
173 self._CHALLENGE_URL, challenge_req,
174 'Logging in', 'Unable to log in')
175
176 if challenge_results is False:
177 return
178
179 login_res = try_get(challenge_results, lambda x: x[0][5], list)
180 if login_res:
181 login_msg = try_get(login_res, lambda x: x[5], compat_str)
182 warn(
183 'Unable to login: %s' % 'Invalid password'
184 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
185 return False
186
187 res = try_get(challenge_results, lambda x: x[0][-1], list)
188 if not res:
189 warn('Unable to extract result entry')
190 return False
191
192 login_challenge = try_get(res, lambda x: x[0][0], list)
193 if login_challenge:
194 challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
195 if challenge_str == 'TWO_STEP_VERIFICATION':
196 # SEND_SUCCESS - TFA code has been successfully sent to phone
197 # QUOTA_EXCEEDED - reached the limit of TFA codes
198 status = try_get(login_challenge, lambda x: x[5], compat_str)
199 if status == 'QUOTA_EXCEEDED':
200 warn('Exceeded the limit of TFA codes, try later')
201 return False
202
203 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
204 if not tl:
205 warn('Unable to extract TL')
206 return False
207
208 tfa_code = self._get_tfa_info('2-step verification code')
209
210 if not tfa_code:
211 warn(
212 'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
213 '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
214 return False
215
216 tfa_code = remove_start(tfa_code, 'G-')
217
218 tfa_req = [
219 user_hash, None, 2, None,
220 [
221 9, None, None, None, None, None, None, None,
222 [None, tfa_code, True, 2]
223 ]]
224
225 tfa_results = req(
226 self._TFA_URL.format(tl), tfa_req,
227 'Submitting TFA code', 'Unable to submit TFA code')
228
229 if tfa_results is False:
230 return False
231
232 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
233 if tfa_res:
234 tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
235 warn(
236 'Unable to finish TFA: %s' % 'Invalid TFA code'
237 if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
238 return False
239
240 check_cookie_url = try_get(
241 tfa_results, lambda x: x[0][-1][2], compat_str)
242 else:
243 CHALLENGES = {
244 'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
245 'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
246 'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
247 }
248 challenge = CHALLENGES.get(
249 challenge_str,
250 '%s returned error %s.' % (self.IE_NAME, challenge_str))
251 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
252 return False
253 else:
254 check_cookie_url = try_get(res, lambda x: x[2], compat_str)
255
256 if not check_cookie_url:
257 warn('Unable to extract CheckCookie URL')
258 return False
259
260 check_cookie_results = self._download_webpage(
261 check_cookie_url, None, 'Checking cookie', fatal=False)
262
263 if check_cookie_results is False:
264 return False
265
266 if 'https://myaccount.google.com/' not in check_cookie_results:
267 warn('Unable to log in')
268 return False
269
270 return True
271
272 def _download_webpage_handle(self, *args, **kwargs):
273 query = kwargs.get('query', {}).copy()
274 query['disable_polymer'] = 'true'
275 kwargs['query'] = query
276 return super(YoutubeBaseInfoExtractor, self)._download_webpage_handle(
277 *args, **compat_kwargs(kwargs))
278
279 def _real_initialize(self):
280 if self._downloader is None:
281 return
282 self._set_language()
283 if not self._login():
284 return
285
286
287 class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
288 # Extract entries from page with "Load more" button
289 def _entries(self, page, playlist_id):
290 more_widget_html = content_html = page
291 for page_num in itertools.count(1):
292 for entry in self._process_page(content_html):
293 yield entry
294
295 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
296 if not mobj:
297 break
298
299 count = 0
300 retries = 3
301 while count <= retries:
302 try:
303 # Downloading page may result in intermittent 5xx HTTP error
304 # that is usually worked around with a retry
305 more = self._download_json(
306 'https://www.youtube.com/%s' % mobj.group('more'), playlist_id,
307 'Downloading page #%s%s'
308 % (page_num, ' (retry #%d)' % count if count else ''),
309 transform_source=uppercase_escape,
310 headers=self._YOUTUBE_CLIENT_HEADERS)
311 break
312 except ExtractorError as e:
313 if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
314 count += 1
315 if count <= retries:
316 continue
317 raise
318
319 content_html = more['content_html']
320 if not content_html.strip():
321 # Some webpages show a "Load more" button but they don't
322 # have more videos
323 break
324 more_widget_html = more['load_more_widget_html']
325
326
327 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
328 def _process_page(self, content):
329 for video_id, video_title in self.extract_videos_from_page(content):
330 yield self.url_result(video_id, 'Youtube', video_id, video_title)
331
332 def extract_videos_from_page_impl(self, video_re, page, ids_in_page, titles_in_page):
333 for mobj in re.finditer(video_re, page):
334 # The link with index 0 is not the first video of the playlist (not sure if still actual)
335 if 'index' in mobj.groupdict() and mobj.group('id') == '0':
336 continue
337 video_id = mobj.group('id')
338 video_title = unescapeHTML(
339 mobj.group('title')) if 'title' in mobj.groupdict() else None
340 if video_title:
341 video_title = video_title.strip()
342 if video_title == 'ā–ŗ Play all':
343 video_title = None
344 try:
345 idx = ids_in_page.index(video_id)
346 if video_title and not titles_in_page[idx]:
347 titles_in_page[idx] = video_title
348 except ValueError:
349 ids_in_page.append(video_id)
350 titles_in_page.append(video_title)
351
352 def extract_videos_from_page(self, page):
353 ids_in_page = []
354 titles_in_page = []
355 self.extract_videos_from_page_impl(
356 self._VIDEO_RE, page, ids_in_page, titles_in_page)
357 return zip(ids_in_page, titles_in_page)
358
359
360 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
361 def _process_page(self, content):
362 for playlist_id in orderedSet(re.findall(
363 r'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
364 content)):
365 yield self.url_result(
366 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
367
368 def _real_extract(self, url):
369 playlist_id = self._match_id(url)
370 webpage = self._download_webpage(url, playlist_id)
371 title = self._og_search_title(webpage, fatal=False)
372 return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
373
374
375 class YoutubeIE(YoutubeBaseInfoExtractor):
376 IE_DESC = 'YouTube.com'
377 _VALID_URL = r"""(?x)^
378 (
379 (?:https?://|//) # http(s):// or protocol-independent URL
380 (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com/|
381 (?:www\.)?deturl\.com/www\.youtube\.com/|
382 (?:www\.)?pwnyoutube\.com/|
383 (?:www\.)?hooktube\.com/|
384 (?:www\.)?yourepeat\.com/|
385 tube\.majestyc\.net/|
386 # Invidious instances taken from https://github.com/omarroth/invidious/wiki/Invidious-Instances
387 (?:(?:www|dev)\.)?invidio\.us/|
388 (?:(?:www|no)\.)?invidiou\.sh/|
389 (?:(?:www|fi|de)\.)?invidious\.snopyta\.org/|
390 (?:www\.)?invidious\.kabi\.tk/|
391 (?:www\.)?invidious\.13ad\.de/|
392 (?:www\.)?invidious\.mastodon\.host/|
393 (?:www\.)?invidious\.nixnet\.xyz/|
394 (?:www\.)?invidious\.drycat\.fr/|
395 (?:www\.)?tube\.poal\.co/|
396 (?:www\.)?vid\.wxzm\.sx/|
397 (?:www\.)?yewtu\.be/|
398 (?:www\.)?yt\.elukerio\.org/|
399 (?:www\.)?yt\.lelux\.fi/|
400 (?:www\.)?invidious\.ggc-project\.de/|
401 (?:www\.)?yt\.maisputain\.ovh/|
402 (?:www\.)?invidious\.13ad\.de/|
403 (?:www\.)?invidious\.toot\.koeln/|
404 (?:www\.)?invidious\.fdn\.fr/|
405 (?:www\.)?watch\.nettohikari\.com/|
406 (?:www\.)?kgg2m7yk5aybusll\.onion/|
407 (?:www\.)?qklhadlycap4cnod\.onion/|
408 (?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion/|
409 (?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion/|
410 (?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion/|
411 (?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion/|
412 (?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p/|
413 (?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion/|
414 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
415 (?:.*?\#/)? # handle anchor (#/) redirect urls
416 (?: # the various things that can precede the ID:
417 (?:(?:v|embed|e)/(?!videoseries)) # v/ or embed/ or e/
418 |(?: # or the v= param in all its forms
419 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
420 (?:\?|\#!?) # the params delimiter ? or # or #!
421 (?:.*?[&;])?? # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
422 v=
423 )
424 ))
425 |(?:
426 youtu\.be| # just youtu.be/xxxx
427 vid\.plus| # or vid.plus/xxxx
428 zwearz\.com/watch| # or zwearz.com/watch/xxxx
429 )/
430 |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
431 )
432 )? # all until now is optional -> you can pass the naked ID
433 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
434 (?!.*?\blist=
435 (?:
436 %(playlist_id)s| # combined list/video URLs are handled by the playlist IE
437 WL # WL are handled by the watch later IE
438 )
439 )
440 (?(1).+)? # if we found the ID, everything can follow
441 $""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
442 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
443 _PLAYER_INFO_RE = (
444 r'/(?P<id>[a-zA-Z0-9_-]{8,})/player_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?/base\.(?P<ext>[a-z]+)$',
445 r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.(?P<ext>[a-z]+)$',
446 )
447 _formats = {
448 '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
449 '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
450 '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
451 '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
452 '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
453 '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
454 '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
455 '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
456 # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
457 '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
458 '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
459 '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
460 '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
461 '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
462 '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
463 '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
464 '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
465 '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
466
467
468 # 3D videos
469 '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
470 '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
471 '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
472 '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
473 '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
474 '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
475 '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
476
477 # Apple HTTP Live Streaming
478 '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
479 '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
480 '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
481 '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
482 '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
483 '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
484 '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
485 '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
486
487 # DASH mp4 video
488 '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
489 '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
490 '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
491 '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
492 '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
493 '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
494 '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
495 '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
496 '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
497 '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
498 '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
499 '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
500
501 # Dash mp4 audio
502 '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
503 '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
504 '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
505 '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
506 '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
507 '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
508 '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
509
510 # Dash webm
511 '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
512 '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
513 '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
514 '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
515 '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
516 '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
517 '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
518 '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
519 '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
520 '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
521 '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
522 '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
523 '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
524 '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
525 '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
526 # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
527 '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
528 '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
529 '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
530 '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
531 '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
532 '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
533
534 # Dash webm audio
535 '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
536 '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
537
538 # Dash webm audio with opus inside
539 '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
540 '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
541 '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
542
543 # RTMP (unnamed)
544 '_rtmp': {'protocol': 'rtmp'},
545
546 # av01 video only formats sometimes served with "unknown" codecs
547 '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
548 '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
549 '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
550 '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
551 }
552 _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
553
554 _GEO_BYPASS = False
555
556 IE_NAME = 'youtube'
557 _TESTS = [
558 {
559 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
560 'info_dict': {
561 'id': 'BaW_jenozKc',
562 'ext': 'mp4',
563 'title': 'youtube-dl test video "\'/\\Ƥā†­š•',
564 'uploader': 'Philipp Hagemeister',
565 'uploader_id': 'phihag',
566 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
567 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
568 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
569 'upload_date': '20121002',
570 'description': 'test chars: "\'/\\Ƥā†­š•\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
571 'categories': ['Science & Technology'],
572 'tags': ['youtube-dl'],
573 'duration': 10,
574 'view_count': int,
575 'like_count': int,
576 'dislike_count': int,
577 'start_time': 1,
578 'end_time': 9,
579 }
580 },
581 {
582 'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY',
583 'note': 'Test generic use_cipher_signature video (#897)',
584 'info_dict': {
585 'id': 'UxxajLWwzqY',
586 'ext': 'mp4',
587 'upload_date': '20120506',
588 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
589 'alt_title': 'I Love It (feat. Charli XCX)',
590 'description': 'md5:19a2f98d9032b9311e686ed039564f63',
591 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
592 'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
593 'iconic ep', 'iconic', 'love', 'it'],
594 'duration': 180,
595 'uploader': 'Icona Pop',
596 'uploader_id': 'IconaPop',
597 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop',
598 'creator': 'Icona Pop',
599 'track': 'I Love It (feat. Charli XCX)',
600 'artist': 'Icona Pop',
601 }
602 },
603 {
604 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
605 'note': 'Test VEVO video with age protection (#956)',
606 'info_dict': {
607 'id': '07FYdnEawAQ',
608 'ext': 'mp4',
609 'upload_date': '20130703',
610 'title': 'Justin Timberlake - Tunnel Vision (Official Music Video) (Explicit)',
611 'alt_title': 'Tunnel Vision',
612 'description': 'md5:07dab3356cde4199048e4c7cd93471e1',
613 'duration': 419,
614 'uploader': 'justintimberlakeVEVO',
615 'uploader_id': 'justintimberlakeVEVO',
616 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
617 'creator': 'Justin Timberlake',
618 'track': 'Tunnel Vision',
619 'artist': 'Justin Timberlake',
620 'age_limit': 18,
621 }
622 },
623 {
624 'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
625 'note': 'Embed-only video (#1746)',
626 'info_dict': {
627 'id': 'yZIXLfi8CZQ',
628 'ext': 'mp4',
629 'upload_date': '20120608',
630 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
631 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
632 'uploader': 'SET India',
633 'uploader_id': 'setindia',
634 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
635 'age_limit': 18,
636 }
637 },
638 {
639 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
640 'note': 'Use the first video ID in the URL',
641 'info_dict': {
642 'id': 'BaW_jenozKc',
643 'ext': 'mp4',
644 'title': 'youtube-dl test video "\'/\\Ƥā†­š•',
645 'uploader': 'Philipp Hagemeister',
646 'uploader_id': 'phihag',
647 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
648 'upload_date': '20121002',
649 'description': 'test chars: "\'/\\Ƥā†­š•\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
650 'categories': ['Science & Technology'],
651 'tags': ['youtube-dl'],
652 'duration': 10,
653 'view_count': int,
654 'like_count': int,
655 'dislike_count': int,
656 },
657 'params': {
658 'skip_download': True,
659 },
660 },
661 {
662 'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
663 'note': '256k DASH audio (format 141) via DASH manifest',
664 'info_dict': {
665 'id': 'a9LDPn-MO4I',
666 'ext': 'm4a',
667 'upload_date': '20121002',
668 'uploader_id': '8KVIDEO',
669 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
670 'description': '',
671 'uploader': '8KVIDEO',
672 'title': 'UHDTV TEST 8K VIDEO.mp4'
673 },
674 'params': {
675 'youtube_include_dash_manifest': True,
676 'format': '141',
677 },
678 'skip': 'format 141 not served anymore',
679 },
680 # DASH manifest with encrypted signature
681 {
682 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
683 'info_dict': {
684 'id': 'IB3lcPjvWLA',
685 'ext': 'm4a',
686 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
687 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
688 'duration': 244,
689 'uploader': 'AfrojackVEVO',
690 'uploader_id': 'AfrojackVEVO',
691 'upload_date': '20131011',
692 },
693 'params': {
694 'youtube_include_dash_manifest': True,
695 'format': '141/bestaudio[ext=m4a]',
696 },
697 },
698 # JS player signature function name containing $
699 {
700 'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
701 'info_dict': {
702 'id': 'nfWlot6h_JM',
703 'ext': 'm4a',
704 'title': 'Taylor Swift - Shake It Off',
705 'description': 'md5:307195cd21ff7fa352270fe884570ef0',
706 'duration': 242,
707 'uploader': 'TaylorSwiftVEVO',
708 'uploader_id': 'TaylorSwiftVEVO',
709 'upload_date': '20140818',
710 },
711 'params': {
712 'youtube_include_dash_manifest': True,
713 'format': '141/bestaudio[ext=m4a]',
714 },
715 },
716 # Controversy video
717 {
718 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
719 'info_dict': {
720 'id': 'T4XJQO3qol8',
721 'ext': 'mp4',
722 'duration': 219,
723 'upload_date': '20100909',
724 'uploader': 'Amazing Atheist',
725 'uploader_id': 'TheAmazingAtheist',
726 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
727 'title': 'Burning Everyone\'s Koran',
728 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
729 }
730 },
731 # Normal age-gate video (No vevo, embed allowed)
732 {
733 'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
734 'info_dict': {
735 'id': 'HtVdAasjOgU',
736 'ext': 'mp4',
737 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
738 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
739 'duration': 142,
740 'uploader': 'The Witcher',
741 'uploader_id': 'WitcherGame',
742 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
743 'upload_date': '20140605',
744 'age_limit': 18,
745 },
746 },
747 # Age-gate video with encrypted signature
748 {
749 'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
750 'info_dict': {
751 'id': '6kLq3WMV1nU',
752 'ext': 'mp4',
753 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
754 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
755 'duration': 246,
756 'uploader': 'LloydVEVO',
757 'uploader_id': 'LloydVEVO',
758 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
759 'upload_date': '20110629',
760 'age_limit': 18,
761 },
762 },
763 # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
764 # YouTube Red ad is not captured for creator
765 {
766 'url': '__2ABJjxzNo',
767 'info_dict': {
768 'id': '__2ABJjxzNo',
769 'ext': 'mp4',
770 'duration': 266,
771 'upload_date': '20100430',
772 'uploader_id': 'deadmau5',
773 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
774 'creator': 'Dada Life, deadmau5',
775 'description': 'md5:12c56784b8032162bb936a5f76d55360',
776 'uploader': 'deadmau5',
777 'title': 'Deadmau5 - Some Chords (HD)',
778 'alt_title': 'This Machine Kills Some Chords',
779 },
780 'expected_warnings': [
781 'DASH manifest missing',
782 ]
783 },
784 # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
785 {
786 'url': 'lqQg6PlCWgI',
787 'info_dict': {
788 'id': 'lqQg6PlCWgI',
789 'ext': 'mp4',
790 'duration': 6085,
791 'upload_date': '20150827',
792 'uploader_id': 'olympic',
793 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
794 'description': 'HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
795 'uploader': 'Olympic',
796 'title': 'Hockey - Women - GER-AUS - London 2012 Olympic Games',
797 },
798 'params': {
799 'skip_download': 'requires avconv',
800 }
801 },
802 # Non-square pixels
803 {
804 'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
805 'info_dict': {
806 'id': '_b-2C3KPAM0',
807 'ext': 'mp4',
808 'stretched_ratio': 16 / 9.,
809 'duration': 85,
810 'upload_date': '20110310',
811 'uploader_id': 'AllenMeow',
812 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
813 'description': 'made by Wacom from Korea | 字幕&åŠ ę²¹ę·»é†‹ by TY\'s Allen | ę„Ÿč¬heylisa00cavey1001同å­øē†±ęƒ…ęä¾›ę¢—åŠēæ»č­Æ',
814 'uploader': 'å­«į„‹į„…',
815 'title': '[A-made] č®Šę…‹å¦å­—å¹•ē‰ˆ å¤Ŗ妍 ęˆ‘å°±ę˜Æ這ęØ£ēš„äŗŗ',
816 },
817 },
818 # url_encoded_fmt_stream_map is empty string
819 {
820 'url': 'qEJwOuvDf7I',
821 'info_dict': {
822 'id': 'qEJwOuvDf7I',
823 'ext': 'webm',
824 'title': 'ŠžŠ±ŃŃƒŠ¶Š“ŠµŠ½ŠøŠµ суŠ“ŠµŠ±Š½Š¾Š¹ ŠæрŠ°ŠŗтŠøŠŗŠø ŠæŠ¾ Š²Ń‹Š±Š¾Ń€Š°Š¼ 14 сŠµŠ½Ń‚яŠ±Ń€Ń 2014 Š³Š¾Š“Š° Š² Š”Š°Š½Šŗт-ŠŸŠµŃ‚ŠµŃ€Š±ŃƒŃ€Š³Šµ',
825 'description': '',
826 'upload_date': '20150404',
827 'uploader_id': 'spbelect',
828 'uploader': 'ŠŠ°Š±Š»ŃŽŠ“Š°Ń‚ŠµŠ»Šø ŠŸŠµŃ‚ŠµŃ€Š±ŃƒŃ€Š³Š°',
829 },
830 'params': {
831 'skip_download': 'requires avconv',
832 },
833 'skip': 'This live event has ended.',
834 },
835 # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
836 {
837 'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
838 'info_dict': {
839 'id': 'FIl7x6_3R5Y',
840 'ext': 'webm',
841 'title': 'md5:7b81415841e02ecd4313668cde88737a',
842 'description': 'md5:116377fd2963b81ec4ce64b542173306',
843 'duration': 220,
844 'upload_date': '20150625',
845 'uploader_id': 'dorappi2000',
846 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
847 'uploader': 'dorappi2000',
848 'formats': 'mincount:31',
849 },
850 'skip': 'not actual anymore',
851 },
852 # DASH manifest with segment_list
853 {
854 'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
855 'md5': '8ce563a1d667b599d21064e982ab9e31',
856 'info_dict': {
857 'id': 'CsmdDsKjzN8',
858 'ext': 'mp4',
859 'upload_date': '20150501', # According to '<meta itemprop="datePublished"', but in other places it's 20150510
860 'uploader': 'Airtek',
861 'description': 'RetransmisiĆ³n en directo de la XVIII media maratĆ³n de Zaragoza.',
862 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
863 'title': 'RetransmisiĆ³n XVIII Media maratĆ³n Zaragoza 2015',
864 },
865 'params': {
866 'youtube_include_dash_manifest': True,
867 'format': '135', # bestvideo
868 },
869 'skip': 'This live event has ended.',
870 },
871 {
872 # Multifeed videos (multiple cameras), URL is for Main Camera
873 'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
874 'info_dict': {
875 'id': 'jqWvoWXjCVs',
876 'title': 'teamPGP: Rocket League Noob Stream',
877 'description': 'md5:dc7872fb300e143831327f1bae3af010',
878 },
879 'playlist': [{
880 'info_dict': {
881 'id': 'jqWvoWXjCVs',
882 'ext': 'mp4',
883 'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
884 'description': 'md5:dc7872fb300e143831327f1bae3af010',
885 'duration': 7335,
886 'upload_date': '20150721',
887 'uploader': 'Beer Games Beer',
888 'uploader_id': 'beergamesbeer',
889 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
890 'license': 'Standard YouTube License',
891 },
892 }, {
893 'info_dict': {
894 'id': '6h8e8xoXJzg',
895 'ext': 'mp4',
896 'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
897 'description': 'md5:dc7872fb300e143831327f1bae3af010',
898 'duration': 7337,
899 'upload_date': '20150721',
900 'uploader': 'Beer Games Beer',
901 'uploader_id': 'beergamesbeer',
902 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
903 'license': 'Standard YouTube License',
904 },
905 }, {
906 'info_dict': {
907 'id': 'PUOgX5z9xZw',
908 'ext': 'mp4',
909 'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
910 'description': 'md5:dc7872fb300e143831327f1bae3af010',
911 'duration': 7337,
912 'upload_date': '20150721',
913 'uploader': 'Beer Games Beer',
914 'uploader_id': 'beergamesbeer',
915 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
916 'license': 'Standard YouTube License',
917 },
918 }, {
919 'info_dict': {
920 'id': 'teuwxikvS5k',
921 'ext': 'mp4',
922 'title': 'teamPGP: Rocket League Noob Stream (zim)',
923 'description': 'md5:dc7872fb300e143831327f1bae3af010',
924 'duration': 7334,
925 'upload_date': '20150721',
926 'uploader': 'Beer Games Beer',
927 'uploader_id': 'beergamesbeer',
928 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
929 'license': 'Standard YouTube License',
930 },
931 }],
932 'params': {
933 'skip_download': True,
934 },
935 'skip': 'This video is not available.',
936 },
937 {
938 # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
939 'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
940 'info_dict': {
941 'id': 'gVfLd0zydlo',
942 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
943 },
944 'playlist_count': 2,
945 'skip': 'Not multifeed anymore',
946 },
947 {
948 'url': 'https://vid.plus/FlRa-iH7PGw',
949 'only_matching': True,
950 },
951 {
952 'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
953 'only_matching': True,
954 },
955 {
956 # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
957 # Also tests cut-off URL expansion in video description (see
958 # https://github.com/ytdl-org/youtube-dl/issues/1892,
959 # https://github.com/ytdl-org/youtube-dl/issues/8164)
960 'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
961 'info_dict': {
962 'id': 'lsguqyKfVQg',
963 'ext': 'mp4',
964 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
965 'alt_title': 'Dark Walk - Position Music',
966 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
967 'duration': 133,
968 'upload_date': '20151119',
969 'uploader_id': 'IronSoulElf',
970 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
971 'uploader': 'IronSoulElf',
972 'creator': 'Todd Haberman, Daniel Law Heath and Aaron Kaplan',
973 'track': 'Dark Walk - Position Music',
974 'artist': 'Todd Haberman, Daniel Law Heath and Aaron Kaplan',
975 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
976 },
977 'params': {
978 'skip_download': True,
979 },
980 },
981 {
982 # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
983 'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
984 'only_matching': True,
985 },
986 {
987 # Video with yt:stretch=17:0
988 'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
989 'info_dict': {
990 'id': 'Q39EVAstoRM',
991 'ext': 'mp4',
992 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
993 'description': 'md5:ee18a25c350637c8faff806845bddee9',
994 'upload_date': '20151107',
995 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
996 'uploader': 'CH GAMER DROID',
997 },
998 'params': {
999 'skip_download': True,
1000 },
1001 'skip': 'This video does not exist.',
1002 },
1003 {
1004 # Video licensed under Creative Commons
1005 'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1006 'info_dict': {
1007 'id': 'M4gD1WSo5mA',
1008 'ext': 'mp4',
1009 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1010 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1011 'duration': 721,
1012 'upload_date': '20150127',
1013 'uploader_id': 'BerkmanCenter',
1014 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1015 'uploader': 'The Berkman Klein Center for Internet & Society',
1016 'license': 'Creative Commons Attribution license (reuse allowed)',
1017 },
1018 'params': {
1019 'skip_download': True,
1020 },
1021 },
1022 {
1023 # Channel-like uploader_url
1024 'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1025 'info_dict': {
1026 'id': 'eQcmzGIKrzg',
1027 'ext': 'mp4',
1028 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1029 'description': 'md5:dda0d780d5a6e120758d1711d062a867',
1030 'duration': 4060,
1031 'upload_date': '20151119',
1032 'uploader': 'Bernie Sanders',
1033 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1034 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1035 'license': 'Creative Commons Attribution license (reuse allowed)',
1036 },
1037 'params': {
1038 'skip_download': True,
1039 },
1040 },
1041 {
1042 'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1043 'only_matching': True,
1044 },
1045 {
1046 # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1047 'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1048 'only_matching': True,
1049 },
1050 {
1051 # Rental video preview
1052 'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1053 'info_dict': {
1054 'id': 'uGpuVWrhIzE',
1055 'ext': 'mp4',
1056 'title': 'Piku - Trailer',
1057 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1058 'upload_date': '20150811',
1059 'uploader': 'FlixMatrix',
1060 'uploader_id': 'FlixMatrixKaravan',
1061 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1062 'license': 'Standard YouTube License',
1063 },
1064 'params': {
1065 'skip_download': True,
1066 },
1067 'skip': 'This video is not available.',
1068 },
1069 {
1070 # YouTube Red video with episode data
1071 'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1072 'info_dict': {
1073 'id': 'iqKdEhx-dD4',
1074 'ext': 'mp4',
1075 'title': 'Isolation - Mind Field (Ep 1)',
1076 'description': 'md5:46a29be4ceffa65b92d277b93f463c0f',
1077 'duration': 2085,
1078 'upload_date': '20170118',
1079 'uploader': 'Vsauce',
1080 'uploader_id': 'Vsauce',
1081 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1082 'series': 'Mind Field',
1083 'season_number': 1,
1084 'episode_number': 1,
1085 },
1086 'params': {
1087 'skip_download': True,
1088 },
1089 'expected_warnings': [
1090 'Skipping DASH manifest',
1091 ],
1092 },
1093 {
1094 # The following content has been identified by the YouTube community
1095 # as inappropriate or offensive to some audiences.
1096 'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1097 'info_dict': {
1098 'id': '6SJNVb0GnPI',
1099 'ext': 'mp4',
1100 'title': 'Race Differences in Intelligence',
1101 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1102 'duration': 965,
1103 'upload_date': '20140124',
1104 'uploader': 'New Century Foundation',
1105 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1106 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1107 },
1108 'params': {
1109 'skip_download': True,
1110 },
1111 },
1112 {
1113 # itag 212
1114 'url': '1t24XAntNCY',
1115 'only_matching': True,
1116 },
1117 {
1118 # geo restricted to JP
1119 'url': 'sJL6WA-aGkQ',
1120 'only_matching': True,
1121 },
1122 {
1123 'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
1124 'only_matching': True,
1125 },
1126 {
1127 'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1128 'only_matching': True,
1129 },
1130 {
1131 # DRM protected
1132 'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1133 'only_matching': True,
1134 },
1135 {
1136 # Video with unsupported adaptive stream type formats
1137 'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1138 'info_dict': {
1139 'id': 'Z4Vy8R84T1U',
1140 'ext': 'mp4',
1141 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1142 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1143 'duration': 433,
1144 'upload_date': '20130923',
1145 'uploader': 'Amelia Putri Harwita',
1146 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1147 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1148 'formats': 'maxcount:10',
1149 },
1150 'params': {
1151 'skip_download': True,
1152 'youtube_include_dash_manifest': False,
1153 },
1154 'skip': 'not actual anymore',
1155 },
1156 {
1157 # Youtube Music Auto-generated description
1158 'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1159 'info_dict': {
1160 'id': 'MgNrAu2pzNs',
1161 'ext': 'mp4',
1162 'title': 'Voyeur Girl',
1163 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1164 'upload_date': '20190312',
1165 'uploader': 'Stephen - Topic',
1166 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1167 'artist': 'Stephen',
1168 'track': 'Voyeur Girl',
1169 'album': 'it\'s too much love to know my dear',
1170 'release_date': '20190313',
1171 'release_year': 2019,
1172 },
1173 'params': {
1174 'skip_download': True,
1175 },
1176 },
1177 {
1178 # Youtube Music Auto-generated description
1179 # Retrieve 'artist' field from 'Artist:' in video description
1180 # when it is present on youtube music video
1181 'url': 'https://www.youtube.com/watch?v=k0jLE7tTwjY',
1182 'info_dict': {
1183 'id': 'k0jLE7tTwjY',
1184 'ext': 'mp4',
1185 'title': 'Latch Feat. Sam Smith',
1186 'description': 'md5:3cb1e8101a7c85fcba9b4fb41b951335',
1187 'upload_date': '20150110',
1188 'uploader': 'Various Artists - Topic',
1189 'uploader_id': 'UCNkEcmYdjrH4RqtNgh7BZ9w',
1190 'artist': 'Disclosure',
1191 'track': 'Latch Feat. Sam Smith',
1192 'album': 'Latch Featuring Sam Smith',
1193 'release_date': '20121008',
1194 'release_year': 2012,
1195 },
1196 'params': {
1197 'skip_download': True,
1198 },
1199 },
1200 {
1201 # Youtube Music Auto-generated description
1202 # handle multiple artists on youtube music video
1203 'url': 'https://www.youtube.com/watch?v=74qn0eJSjpA',
1204 'info_dict': {
1205 'id': '74qn0eJSjpA',
1206 'ext': 'mp4',
1207 'title': 'Eastside',
1208 'description': 'md5:290516bb73dcbfab0dcc4efe6c3de5f2',
1209 'upload_date': '20180710',
1210 'uploader': 'Benny Blanco - Topic',
1211 'uploader_id': 'UCzqz_ksRu_WkIzmivMdIS7A',
1212 'artist': 'benny blanco, Halsey, Khalid',
1213 'track': 'Eastside',
1214 'album': 'Eastside',
1215 'release_date': '20180713',
1216 'release_year': 2018,
1217 },
1218 'params': {
1219 'skip_download': True,
1220 },
1221 },
1222 {
1223 # Youtube Music Auto-generated description
1224 # handle youtube music video with release_year and no release_date
1225 'url': 'https://www.youtube.com/watch?v=-hcAI0g-f5M',
1226 'info_dict': {
1227 'id': '-hcAI0g-f5M',
1228 'ext': 'mp4',
1229 'title': 'Put It On Me',
1230 'description': 'md5:f6422397c07c4c907c6638e1fee380a5',
1231 'upload_date': '20180426',
1232 'uploader': 'Matt Maeson - Topic',
1233 'uploader_id': 'UCnEkIGqtGcQMLk73Kp-Q5LQ',
1234 'artist': 'Matt Maeson',
1235 'track': 'Put It On Me',
1236 'album': 'The Hearse',
1237 'release_date': None,
1238 'release_year': 2018,
1239 },
1240 'params': {
1241 'skip_download': True,
1242 },
1243 },
1244 {
1245 'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1246 'only_matching': True,
1247 },
1248 {
1249 # invalid -> valid video id redirection
1250 'url': 'DJztXj2GPfl',
1251 'info_dict': {
1252 'id': 'DJztXj2GPfk',
1253 'ext': 'mp4',
1254 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1255 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1256 'upload_date': '20090125',
1257 'uploader': 'Prochorowka',
1258 'uploader_id': 'Prochorowka',
1259 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1260 'artist': 'Panjabi MC',
1261 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1262 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1263 },
1264 'params': {
1265 'skip_download': True,
1266 },
1267 },
1268 {
1269 # empty description results in an empty string
1270 'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1271 'info_dict': {
1272 'id': 'x41yOUIvK2k',
1273 'ext': 'mp4',
1274 'title': 'IMG 3456',
1275 'description': '',
1276 'upload_date': '20170613',
1277 'uploader_id': 'ElevageOrVert',
1278 'uploader': 'ElevageOrVert',
1279 },
1280 'params': {
1281 'skip_download': True,
1282 },
1283 },
1284 ]
1285
1286 def __init__(self, *args, **kwargs):
1287 super(YoutubeIE, self).__init__(*args, **kwargs)
1288 self._player_cache = {}
1289
1290 def report_video_info_webpage_download(self, video_id):
1291 """Report attempt to download video info webpage."""
1292 self.to_screen('%s: Downloading video info webpage' % video_id)
1293
1294 def report_information_extraction(self, video_id):
1295 """Report attempt to extract video information."""
1296 self.to_screen('%s: Extracting video information' % video_id)
1297
1298 def report_unavailable_format(self, video_id, format):
1299 """Report extracted video URL."""
1300 self.to_screen('%s: Format %s not available' % (video_id, format))
1301
1302 def report_rtmp_download(self):
1303 """Indicate the download will use the RTMP protocol."""
1304 self.to_screen('RTMP download detected')
1305
1306 def _signature_cache_id(self, example_sig):
1307 """ Return a string representation of a signature """
1308 return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
1309
1310 @classmethod
1311 def _extract_player_info(cls, player_url):
1312 for player_re in cls._PLAYER_INFO_RE:
1313 id_m = re.search(player_re, player_url)
1314 if id_m:
1315 break
1316 else:
1317 raise ExtractorError('Cannot identify player %r' % player_url)
1318 return id_m.group('ext'), id_m.group('id')
1319
1320 def _extract_signature_function(self, video_id, player_url, example_sig):
1321 player_type, player_id = self._extract_player_info(player_url)
1322
1323 # Read from filesystem cache
1324 func_id = '%s_%s_%s' % (
1325 player_type, player_id, self._signature_cache_id(example_sig))
1326 assert os.path.basename(func_id) == func_id
1327
1328 cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
1329 if cache_spec is not None:
1330 return lambda s: ''.join(s[i] for i in cache_spec)
1331
1332 download_note = (
1333 'Downloading player %s' % player_url
1334 if self._downloader.params.get('verbose') else
1335 'Downloading %s player %s' % (player_type, player_id)
1336 )
1337 if player_type == 'js':
1338 code = self._download_webpage(
1339 player_url, video_id,
1340 note=download_note,
1341 errnote='Download of %s failed' % player_url)
1342 res = self._parse_sig_js(code)
1343 elif player_type == 'swf':
1344 urlh = self._request_webpage(
1345 player_url, video_id,
1346 note=download_note,
1347 errnote='Download of %s failed' % player_url)
1348 code = urlh.read()
1349 res = self._parse_sig_swf(code)
1350 else:
1351 assert False, 'Invalid player type %r' % player_type
1352
1353 test_string = ''.join(map(compat_chr, range(len(example_sig))))
1354 cache_res = res(test_string)
1355 cache_spec = [ord(c) for c in cache_res]
1356
1357 self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
1358 return res
1359
1360 def _print_sig_code(self, func, example_sig):
1361 def gen_sig_code(idxs):
1362 def _genslice(start, end, step):
1363 starts = '' if start == 0 else str(start)
1364 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
1365 steps = '' if step == 1 else (':%d' % step)
1366 return 's[%s%s%s]' % (starts, ends, steps)
1367
1368 step = None
1369 # Quelch pyflakes warnings - start will be set when step is set
1370 start = '(Never used)'
1371 for i, prev in zip(idxs[1:], idxs[:-1]):
1372 if step is not None:
1373 if i - prev == step:
1374 continue
1375 yield _genslice(start, prev, step)
1376 step = None
1377 continue
1378 if i - prev in [-1, 1]:
1379 step = i - prev
1380 start = prev
1381 continue
1382 else:
1383 yield 's[%d]' % prev
1384 if step is None:
1385 yield 's[%d]' % i
1386 else:
1387 yield _genslice(start, i, step)
1388
1389 test_string = ''.join(map(compat_chr, range(len(example_sig))))
1390 cache_res = func(test_string)
1391 cache_spec = [ord(c) for c in cache_res]
1392 expr_code = ' + '.join(gen_sig_code(cache_spec))
1393 signature_id_tuple = '(%s)' % (
1394 ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1395 code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1396 ' return %s\n') % (signature_id_tuple, expr_code)
1397 self.to_screen('Extracted signature function:\n' + code)
1398
1399 def _parse_sig_js(self, jscode):
1400 funcname = self._search_regex(
1401 (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1402 r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1403 r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1404 r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1405 # Obsolete patterns
1406 r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1407 r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
1408 r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1409 r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1410 r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1411 r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1412 r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1413 r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
1414 jscode, 'Initial JS player signature function name', group='sig')
1415
1416 jsi = JSInterpreter(jscode)
1417 initial_function = jsi.extract_function(funcname)
1418 return lambda s: initial_function([s])
1419
1420 def _parse_sig_swf(self, file_contents):
1421 swfi = SWFInterpreter(file_contents)
1422 TARGET_CLASSNAME = 'SignatureDecipher'
1423 searched_class = swfi.extract_class(TARGET_CLASSNAME)
1424 initial_function = swfi.extract_function(searched_class, 'decipher')
1425 return lambda s: initial_function([s])
1426
1427 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1428 """Turn the encrypted s field into a working signature"""
1429
1430 if player_url is None:
1431 raise ExtractorError('Cannot decrypt signature without player_url')
1432
1433 if player_url.startswith('//'):
1434 player_url = 'https:' + player_url
1435 elif not re.match(r'https?://', player_url):
1436 player_url = compat_urlparse.urljoin(
1437 'https://www.youtube.com', player_url)
1438 try:
1439 player_id = (player_url, self._signature_cache_id(s))
1440 if player_id not in self._player_cache:
1441 func = self._extract_signature_function(
1442 video_id, player_url, s
1443 )
1444 self._player_cache[player_id] = func
1445 func = self._player_cache[player_id]
1446 if self._downloader.params.get('youtube_print_sig_code'):
1447 self._print_sig_code(func, s)
1448 return func(s)
1449 except Exception as e:
1450 tb = traceback.format_exc()
1451 raise ExtractorError(
1452 'Signature extraction failed: ' + tb, cause=e)
1453
1454 def _get_subtitles(self, video_id, webpage):
1455 try:
1456 subs_doc = self._download_xml(
1457 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1458 video_id, note=False)
1459 except ExtractorError as err:
1460 self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
1461 return {}
1462
1463 sub_lang_list = {}
1464 for track in subs_doc.findall('track'):
1465 lang = track.attrib['lang_code']
1466 if lang in sub_lang_list:
1467 continue
1468 sub_formats = []
1469 for ext in self._SUBTITLE_FORMATS:
1470 params = compat_urllib_parse_urlencode({
1471 'lang': lang,
1472 'v': video_id,
1473 'fmt': ext,
1474 'name': track.attrib['name'].encode('utf-8'),
1475 })
1476 sub_formats.append({
1477 'url': 'https://www.youtube.com/api/timedtext?' + params,
1478 'ext': ext,
1479 })
1480 sub_lang_list[lang] = sub_formats
1481 if not sub_lang_list:
1482 self._downloader.report_warning('video doesn\'t have subtitles')
1483 return {}
1484 return sub_lang_list
1485
1486 def _get_ytplayer_config(self, video_id, webpage):
1487 patterns = (
1488 # User data may contain arbitrary character sequences that may affect
1489 # JSON extraction with regex, e.g. when '};' is contained the second
1490 # regex won't capture the whole JSON. Yet working around by trying more
1491 # concrete regex first keeping in mind proper quoted string handling
1492 # to be implemented in future that will replace this workaround (see
1493 # https://github.com/ytdl-org/youtube-dl/issues/7468,
1494 # https://github.com/ytdl-org/youtube-dl/pull/7599)
1495 r';ytplayer\.config\s*=\s*({.+?});ytplayer',
1496 r';ytplayer\.config\s*=\s*({.+?});',
1497 )
1498 config = self._search_regex(
1499 patterns, webpage, 'ytplayer.config', default=None)
1500 if config:
1501 return self._parse_json(
1502 uppercase_escape(config), video_id, fatal=False)
1503
1504 def _get_automatic_captions(self, video_id, webpage):
1505 """We need the webpage for getting the captions url, pass it as an
1506 argument to speed up the process."""
1507 self.to_screen('%s: Looking for automatic captions' % video_id)
1508 player_config = self._get_ytplayer_config(video_id, webpage)
1509 err_msg = 'Couldn\'t find automatic captions for %s' % video_id
1510 if not player_config:
1511 self._downloader.report_warning(err_msg)
1512 return {}
1513 try:
1514 args = player_config['args']
1515 caption_url = args.get('ttsurl')
1516 if caption_url:
1517 timestamp = args['timestamp']
1518 # We get the available subtitles
1519 list_params = compat_urllib_parse_urlencode({
1520 'type': 'list',
1521 'tlangs': 1,
1522 'asrs': 1,
1523 })
1524 list_url = caption_url + '&' + list_params
1525 caption_list = self._download_xml(list_url, video_id)
1526 original_lang_node = caption_list.find('track')
1527 if original_lang_node is None:
1528 self._downloader.report_warning('Video doesn\'t have automatic captions')
1529 return {}
1530 original_lang = original_lang_node.attrib['lang_code']
1531 caption_kind = original_lang_node.attrib.get('kind', '')
1532
1533 sub_lang_list = {}
1534 for lang_node in caption_list.findall('target'):
1535 sub_lang = lang_node.attrib['lang_code']
1536 sub_formats = []
1537 for ext in self._SUBTITLE_FORMATS:
1538 params = compat_urllib_parse_urlencode({
1539 'lang': original_lang,
1540 'tlang': sub_lang,
1541 'fmt': ext,
1542 'ts': timestamp,
1543 'kind': caption_kind,
1544 })
1545 sub_formats.append({
1546 'url': caption_url + '&' + params,
1547 'ext': ext,
1548 })
1549 sub_lang_list[sub_lang] = sub_formats
1550 return sub_lang_list
1551
1552 def make_captions(sub_url, sub_langs):
1553 parsed_sub_url = compat_urllib_parse_urlparse(sub_url)
1554 caption_qs = compat_parse_qs(parsed_sub_url.query)
1555 captions = {}
1556 for sub_lang in sub_langs:
1557 sub_formats = []
1558 for ext in self._SUBTITLE_FORMATS:
1559 caption_qs.update({
1560 'tlang': [sub_lang],
1561 'fmt': [ext],
1562 })
1563 sub_url = compat_urlparse.urlunparse(parsed_sub_url._replace(
1564 query=compat_urllib_parse_urlencode(caption_qs, True)))
1565 sub_formats.append({
1566 'url': sub_url,
1567 'ext': ext,
1568 })
1569 captions[sub_lang] = sub_formats
1570 return captions
1571
1572 # New captions format as of 22.06.2017
1573 player_response = args.get('player_response')
1574 if player_response and isinstance(player_response, compat_str):
1575 player_response = self._parse_json(
1576 player_response, video_id, fatal=False)
1577 if player_response:
1578 renderer = player_response['captions']['playerCaptionsTracklistRenderer']
1579 base_url = renderer['captionTracks'][0]['baseUrl']
1580 sub_lang_list = []
1581 for lang in renderer['translationLanguages']:
1582 lang_code = lang.get('languageCode')
1583 if lang_code:
1584 sub_lang_list.append(lang_code)
1585 return make_captions(base_url, sub_lang_list)
1586
1587 # Some videos don't provide ttsurl but rather caption_tracks and
1588 # caption_translation_languages (e.g. 20LmZk1hakA)
1589 # Does not used anymore as of 22.06.2017
1590 caption_tracks = args['caption_tracks']
1591 caption_translation_languages = args['caption_translation_languages']
1592 caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
1593 sub_lang_list = []
1594 for lang in caption_translation_languages.split(','):
1595 lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
1596 sub_lang = lang_qs.get('lc', [None])[0]
1597 if sub_lang:
1598 sub_lang_list.append(sub_lang)
1599 return make_captions(caption_url, sub_lang_list)
1600 # An extractor error can be raise by the download process if there are
1601 # no automatic captions but there are subtitles
1602 except (KeyError, IndexError, ExtractorError):
1603 self._downloader.report_warning(err_msg)
1604 return {}
1605
1606 def _mark_watched(self, video_id, video_info, player_response):
1607 playback_url = url_or_none(try_get(
1608 player_response,
1609 lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']) or try_get(
1610 video_info, lambda x: x['videostats_playback_base_url'][0]))
1611 if not playback_url:
1612 return
1613 parsed_playback_url = compat_urlparse.urlparse(playback_url)
1614 qs = compat_urlparse.parse_qs(parsed_playback_url.query)
1615
1616 # cpn generation algorithm is reverse engineered from base.js.
1617 # In fact it works even with dummy cpn.
1618 CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1619 cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
1620
1621 qs.update({
1622 'ver': ['2'],
1623 'cpn': [cpn],
1624 })
1625 playback_url = compat_urlparse.urlunparse(
1626 parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
1627
1628 self._download_webpage(
1629 playback_url, video_id, 'Marking watched',
1630 'Unable to mark watched', fatal=False)
1631
1632 @staticmethod
1633 def _extract_urls(webpage):
1634 # Embedded YouTube player
1635 entries = [
1636 unescapeHTML(mobj.group('url'))
1637 for mobj in re.finditer(r'''(?x)
1638 (?:
1639 <iframe[^>]+?src=|
1640 data-video-url=|
1641 <embed[^>]+?src=|
1642 embedSWF\(?:\s*|
1643 <object[^>]+data=|
1644 new\s+SWFObject\(
1645 )
1646 (["\'])
1647 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1648 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1649 \1''', webpage)]
1650
1651 # lazyYT YouTube embed
1652 entries.extend(list(map(
1653 unescapeHTML,
1654 re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
1655
1656 # Wordpress "YouTube Video Importer" plugin
1657 matches = re.findall(r'''(?x)<div[^>]+
1658 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1659 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1660 entries.extend(m[-1] for m in matches)
1661
1662 return entries
1663
1664 @staticmethod
1665 def _extract_url(webpage):
1666 urls = YoutubeIE._extract_urls(webpage)
1667 return urls[0] if urls else None
1668
1669 @classmethod
1670 def extract_id(cls, url):
1671 mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1672 if mobj is None:
1673 raise ExtractorError('Invalid URL: %s' % url)
1674 video_id = mobj.group(2)
1675 return video_id
1676
1677 def _extract_chapters_from_json(self, webpage, video_id, duration):
1678 if not webpage:
1679 return
1680 player = self._parse_json(
1681 self._search_regex(
1682 r'RELATED_PLAYER_ARGS["\']\s*:\s*({.+})\s*,?\s*\n', webpage,
1683 'player args', default='{}'),
1684 video_id, fatal=False)
1685 if not player or not isinstance(player, dict):
1686 return
1687 watch_next_response = player.get('watch_next_response')
1688 if not isinstance(watch_next_response, compat_str):
1689 return
1690 response = self._parse_json(watch_next_response, video_id, fatal=False)
1691 if not response or not isinstance(response, dict):
1692 return
1693 chapters_list = try_get(
1694 response,
1695 lambda x: x['playerOverlays']
1696 ['playerOverlayRenderer']
1697 ['decoratedPlayerBarRenderer']
1698 ['decoratedPlayerBarRenderer']
1699 ['playerBar']
1700 ['chapteredPlayerBarRenderer']
1701 ['chapters'],
1702 list)
1703 if not chapters_list:
1704 return
1705
1706 def chapter_time(chapter):
1707 return float_or_none(
1708 try_get(
1709 chapter,
1710 lambda x: x['chapterRenderer']['timeRangeStartMillis'],
1711 int),
1712 scale=1000)
1713 chapters = []
1714 for next_num, chapter in enumerate(chapters_list, start=1):
1715 start_time = chapter_time(chapter)
1716 if start_time is None:
1717 continue
1718 end_time = (chapter_time(chapters_list[next_num])
1719 if next_num < len(chapters_list) else duration)
1720 if end_time is None:
1721 continue
1722 title = try_get(
1723 chapter, lambda x: x['chapterRenderer']['title']['simpleText'],
1724 compat_str)
1725 chapters.append({
1726 'start_time': start_time,
1727 'end_time': end_time,
1728 'title': title,
1729 })
1730 return chapters
1731
1732 @staticmethod
1733 def _extract_chapters_from_description(description, duration):
1734 if not description:
1735 return None
1736 chapter_lines = re.findall(
1737 r'(?:^|<br\s*/>)([^<]*<a[^>]+onclick=["\']yt\.www\.watch\.player\.seekTo[^>]+>(\d{1,2}:\d{1,2}(?::\d{1,2})?)</a>[^>]*)(?=$|<br\s*/>)',
1738 description)
1739 if not chapter_lines:
1740 return None
1741 chapters = []
1742 for next_num, (chapter_line, time_point) in enumerate(
1743 chapter_lines, start=1):
1744 start_time = parse_duration(time_point)
1745 if start_time is None:
1746 continue
1747 if start_time > duration:
1748 break
1749 end_time = (duration if next_num == len(chapter_lines)
1750 else parse_duration(chapter_lines[next_num][1]))
1751 if end_time is None:
1752 continue
1753 if end_time > duration:
1754 end_time = duration
1755 if start_time > end_time:
1756 break
1757 chapter_title = re.sub(
1758 r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
1759 chapter_title = re.sub(r'\s+', ' ', chapter_title)
1760 chapters.append({
1761 'start_time': start_time,
1762 'end_time': end_time,
1763 'title': chapter_title,
1764 })
1765 return chapters
1766
1767 def _extract_chapters(self, webpage, description, video_id, duration):
1768 return (self._extract_chapters_from_json(webpage, video_id, duration)
1769 or self._extract_chapters_from_description(description, duration))
1770
1771 def _real_extract(self, url):
1772 url, smuggled_data = unsmuggle_url(url, {})
1773
1774 proto = (
1775 'http' if self._downloader.params.get('prefer_insecure', False)
1776 else 'https')
1777
1778 start_time = None
1779 end_time = None
1780 parsed_url = compat_urllib_parse_urlparse(url)
1781 for component in [parsed_url.fragment, parsed_url.query]:
1782 query = compat_parse_qs(component)
1783 if start_time is None and 't' in query:
1784 start_time = parse_duration(query['t'][0])
1785 if start_time is None and 'start' in query:
1786 start_time = parse_duration(query['start'][0])
1787 if end_time is None and 'end' in query:
1788 end_time = parse_duration(query['end'][0])
1789
1790 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1791 mobj = re.search(self._NEXT_URL_RE, url)
1792 if mobj:
1793 url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
1794 video_id = self.extract_id(url)
1795
1796 # Get video webpage
1797 url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1798 video_webpage, urlh = self._download_webpage_handle(url, video_id)
1799
1800 qs = compat_parse_qs(compat_urllib_parse_urlparse(urlh.geturl()).query)
1801 video_id = qs.get('v', [None])[0] or video_id
1802
1803 # Attempt to extract SWF player URL
1804 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1805 if mobj is not None:
1806 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1807 else:
1808 player_url = None
1809
1810 dash_mpds = []
1811
1812 def add_dash_mpd(video_info):
1813 dash_mpd = video_info.get('dashmpd')
1814 if dash_mpd and dash_mpd[0] not in dash_mpds:
1815 dash_mpds.append(dash_mpd[0])
1816
1817 def add_dash_mpd_pr(pl_response):
1818 dash_mpd = url_or_none(try_get(
1819 pl_response, lambda x: x['streamingData']['dashManifestUrl'],
1820 compat_str))
1821 if dash_mpd and dash_mpd not in dash_mpds:
1822 dash_mpds.append(dash_mpd)
1823
1824 is_live = None
1825 view_count = None
1826
1827 def extract_view_count(v_info):
1828 return int_or_none(try_get(v_info, lambda x: x['view_count'][0]))
1829
1830 def extract_player_response(player_response, video_id):
1831 pl_response = str_or_none(player_response)
1832 if not pl_response:
1833 return
1834 pl_response = self._parse_json(pl_response, video_id, fatal=False)
1835 if isinstance(pl_response, dict):
1836 add_dash_mpd_pr(pl_response)
1837 return pl_response
1838
1839 player_response = {}
1840
1841 # Get video info
1842 video_info = {}
1843 embed_webpage = None
1844 if (self._og_search_property('restrictions:age', video_webpage, default=None) == '18+'
1845 or re.search(r'player-age-gate-content">', video_webpage) is not None):
1846 age_gate = True
1847 # We simulate the access to the video from www.youtube.com/v/{video_id}
1848 # this can be viewed without login into Youtube
1849 url = proto + '://www.youtube.com/embed/%s' % video_id
1850 embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
1851 data = compat_urllib_parse_urlencode({
1852 'video_id': video_id,
1853 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1854 'sts': self._search_regex(
1855 r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
1856 })
1857 video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1858 try:
1859 video_info_webpage = self._download_webpage(
1860 video_info_url, video_id,
1861 note='Refetching age-gated info webpage',
1862 errnote='unable to download video info webpage')
1863 except ExtractorError:
1864 video_info_webpage = None
1865 if video_info_webpage:
1866 video_info = compat_parse_qs(video_info_webpage)
1867 pl_response = video_info.get('player_response', [None])[0]
1868 player_response = extract_player_response(pl_response, video_id)
1869 add_dash_mpd(video_info)
1870 view_count = extract_view_count(video_info)
1871 else:
1872 age_gate = False
1873 # Try looking directly into the video webpage
1874 ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
1875 if ytplayer_config:
1876 args = ytplayer_config['args']
1877 if args.get('url_encoded_fmt_stream_map') or args.get('hlsvp'):
1878 # Convert to the same format returned by compat_parse_qs
1879 video_info = dict((k, [v]) for k, v in args.items())
1880 add_dash_mpd(video_info)
1881 # Rental video is not rented but preview is available (e.g.
1882 # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
1883 # https://github.com/ytdl-org/youtube-dl/issues/10532)
1884 if not video_info and args.get('ypc_vid'):
1885 return self.url_result(
1886 args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
1887 if args.get('livestream') == '1' or args.get('live_playback') == 1:
1888 is_live = True
1889 if not player_response:
1890 player_response = extract_player_response(args.get('player_response'), video_id)
1891 if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
1892 add_dash_mpd_pr(player_response)
1893
1894 def extract_unavailable_message():
1895 messages = []
1896 for tag, kind in (('h1', 'message'), ('div', 'submessage')):
1897 msg = self._html_search_regex(
1898 r'(?s)<{tag}[^>]+id=["\']unavailable-{kind}["\'][^>]*>(.+?)</{tag}>'.format(tag=tag, kind=kind),
1899 video_webpage, 'unavailable %s' % kind, default=None)
1900 if msg:
1901 messages.append(msg)
1902 if messages:
1903 return '\n'.join(messages)
1904
1905 if not video_info and not player_response:
1906 unavailable_message = extract_unavailable_message()
1907 if not unavailable_message:
1908 unavailable_message = 'Unable to extract video data'
1909 raise ExtractorError(
1910 'YouTube said: %s' % unavailable_message, expected=True, video_id=video_id)
1911
1912 if not isinstance(video_info, dict):
1913 video_info = {}
1914
1915 video_details = try_get(
1916 player_response, lambda x: x['videoDetails'], dict) or {}
1917
1918 microformat = try_get(
1919 player_response, lambda x: x['microformat']['playerMicroformatRenderer'], dict) or {}
1920
1921 video_title = video_info.get('title', [None])[0] or video_details.get('title')
1922 if not video_title:
1923 self._downloader.report_warning('Unable to extract video title')
1924 video_title = '_'
1925
1926 description_original = video_description = get_element_by_id("eow-description", video_webpage)
1927 if video_description:
1928
1929 def replace_url(m):
1930 redir_url = compat_urlparse.urljoin(url, m.group(1))
1931 parsed_redir_url = compat_urllib_parse_urlparse(redir_url)
1932 if re.search(r'^(?:www\.)?(?:youtube(?:-nocookie)?\.com|youtu\.be)$', parsed_redir_url.netloc) and parsed_redir_url.path == '/redirect':
1933 qs = compat_parse_qs(parsed_redir_url.query)
1934 q = qs.get('q')
1935 if q and q[0]:
1936 return q[0]
1937 return redir_url
1938
1939 description_original = video_description = re.sub(r'''(?x)
1940 <a\s+
1941 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1942 (?:title|href)="([^"]+)"\s+
1943 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1944 class="[^"]*"[^>]*>
1945 [^<]+\.{3}\s*
1946 </a>
1947 ''', replace_url, video_description)
1948 video_description = clean_html(video_description)
1949 else:
1950 video_description = video_details.get('shortDescription')
1951 if video_description is None:
1952 video_description = self._html_search_meta('description', video_webpage)
1953
1954 if not smuggled_data.get('force_singlefeed', False):
1955 if not self._downloader.params.get('noplaylist'):
1956 multifeed_metadata_list = try_get(
1957 player_response,
1958 lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
1959 compat_str) or try_get(
1960 video_info, lambda x: x['multifeed_metadata_list'][0], compat_str)
1961 if multifeed_metadata_list:
1962 entries = []
1963 feed_ids = []
1964 for feed in multifeed_metadata_list.split(','):
1965 # Unquote should take place before split on comma (,) since textual
1966 # fields may contain comma as well (see
1967 # https://github.com/ytdl-org/youtube-dl/issues/8536)
1968 feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
1969
1970 def feed_entry(name):
1971 return try_get(feed_data, lambda x: x[name][0], compat_str)
1972
1973 feed_id = feed_entry('id')
1974 if not feed_id:
1975 continue
1976 feed_title = feed_entry('title')
1977 title = video_title
1978 if feed_title:
1979 title += ' (%s)' % feed_title
1980 entries.append({
1981 '_type': 'url_transparent',
1982 'ie_key': 'Youtube',
1983 'url': smuggle_url(
1984 '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
1985 {'force_singlefeed': True}),
1986 'title': title,
1987 })
1988 feed_ids.append(feed_id)
1989 self.to_screen(
1990 'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1991 % (', '.join(feed_ids), video_id))
1992 return self.playlist_result(entries, video_id, video_title, video_description)
1993 else:
1994 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1995
1996 if view_count is None:
1997 view_count = extract_view_count(video_info)
1998 if view_count is None and video_details:
1999 view_count = int_or_none(video_details.get('viewCount'))
2000 if view_count is None and microformat:
2001 view_count = int_or_none(microformat.get('viewCount'))
2002
2003 if is_live is None:
2004 is_live = bool_or_none(video_details.get('isLive'))
2005
2006 # Check for "rental" videos
2007 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
2008 raise ExtractorError('"rental" videos not supported. See https://github.com/ytdl-org/youtube-dl/issues/359 for more information.', expected=True)
2009
2010 def _extract_filesize(media_url):
2011 return int_or_none(self._search_regex(
2012 r'\bclen[=/](\d+)', media_url, 'filesize', default=None))
2013
2014 streaming_formats = try_get(player_response, lambda x: x['streamingData']['formats'], list) or []
2015 streaming_formats.extend(try_get(player_response, lambda x: x['streamingData']['adaptiveFormats'], list) or [])
2016
2017 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
2018 self.report_rtmp_download()
2019 formats = [{
2020 'format_id': '_rtmp',
2021 'protocol': 'rtmp',
2022 'url': video_info['conn'][0],
2023 'player_url': player_url,
2024 }]
2025 elif not is_live and (streaming_formats or len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1):
2026 encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
2027 if 'rtmpe%3Dyes' in encoded_url_map:
2028 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/ytdl-org/youtube-dl/issues/343 for more information.', expected=True)
2029 formats = []
2030 formats_spec = {}
2031 fmt_list = video_info.get('fmt_list', [''])[0]
2032 if fmt_list:
2033 for fmt in fmt_list.split(','):
2034 spec = fmt.split('/')
2035 if len(spec) > 1:
2036 width_height = spec[1].split('x')
2037 if len(width_height) == 2:
2038 formats_spec[spec[0]] = {
2039 'resolution': spec[1],
2040 'width': int_or_none(width_height[0]),
2041 'height': int_or_none(width_height[1]),
2042 }
2043 for fmt in streaming_formats:
2044 itag = str_or_none(fmt.get('itag'))
2045 if not itag:
2046 continue
2047 quality = fmt.get('quality')
2048 quality_label = fmt.get('qualityLabel') or quality
2049 formats_spec[itag] = {
2050 'asr': int_or_none(fmt.get('audioSampleRate')),
2051 'filesize': int_or_none(fmt.get('contentLength')),
2052 'format_note': quality_label,
2053 'fps': int_or_none(fmt.get('fps')),
2054 'height': int_or_none(fmt.get('height')),
2055 # bitrate for itag 43 is always 2147483647
2056 'tbr': float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000) if itag != '43' else None,
2057 'width': int_or_none(fmt.get('width')),
2058 }
2059
2060 for fmt in streaming_formats:
2061 if fmt.get('drmFamilies') or fmt.get('drm_families'):
2062 continue
2063 url = url_or_none(fmt.get('url'))
2064
2065 if not url:
2066 cipher = fmt.get('cipher') or fmt.get('signatureCipher')
2067 if not cipher:
2068 continue
2069 url_data = compat_parse_qs(cipher)
2070 url = url_or_none(try_get(url_data, lambda x: x['url'][0], compat_str))
2071 if not url:
2072 continue
2073 else:
2074 cipher = None
2075 url_data = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
2076
2077 stream_type = int_or_none(try_get(url_data, lambda x: x['stream_type'][0]))
2078 # Unsupported FORMAT_STREAM_TYPE_OTF
2079 if stream_type == 3:
2080 continue
2081
2082 format_id = fmt.get('itag') or url_data['itag'][0]
2083 if not format_id:
2084 continue
2085 format_id = compat_str(format_id)
2086
2087 if cipher:
2088 if 's' in url_data or self._downloader.params.get('youtube_include_dash_manifest', True):
2089 ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
2090 jsplayer_url_json = self._search_regex(
2091 ASSETS_RE,
2092 embed_webpage if age_gate else video_webpage,
2093 'JS player URL (1)', default=None)
2094 if not jsplayer_url_json and not age_gate:
2095 # We need the embed website after all
2096 if embed_webpage is None:
2097 embed_url = proto + '://www.youtube.com/embed/%s' % video_id
2098 embed_webpage = self._download_webpage(
2099 embed_url, video_id, 'Downloading embed webpage')
2100 jsplayer_url_json = self._search_regex(
2101 ASSETS_RE, embed_webpage, 'JS player URL')
2102
2103 player_url = json.loads(jsplayer_url_json)
2104 if player_url is None:
2105 player_url_json = self._search_regex(
2106 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
2107 video_webpage, 'age gate player URL')
2108 player_url = json.loads(player_url_json)
2109
2110 if 'sig' in url_data:
2111 url += '&signature=' + url_data['sig'][0]
2112 elif 's' in url_data:
2113 encrypted_sig = url_data['s'][0]
2114
2115 if self._downloader.params.get('verbose'):
2116 if player_url is None:
2117 player_desc = 'unknown'
2118 else:
2119 player_type, player_version = self._extract_player_info(player_url)
2120 player_desc = '%s player %s' % ('flash' if player_type == 'swf' else 'html5', player_version)
2121 parts_sizes = self._signature_cache_id(encrypted_sig)
2122 self.to_screen('{%s} signature length %s, %s' %
2123 (format_id, parts_sizes, player_desc))
2124
2125 signature = self._decrypt_signature(
2126 encrypted_sig, video_id, player_url, age_gate)
2127 sp = try_get(url_data, lambda x: x['sp'][0], compat_str) or 'signature'
2128 url += '&%s=%s' % (sp, signature)
2129 if 'ratebypass' not in url:
2130 url += '&ratebypass=yes'
2131
2132 dct = {
2133 'format_id': format_id,
2134 'url': url,
2135 'player_url': player_url,
2136 }
2137 if format_id in self._formats:
2138 dct.update(self._formats[format_id])
2139 if format_id in formats_spec:
2140 dct.update(formats_spec[format_id])
2141
2142 # Some itags are not included in DASH manifest thus corresponding formats will
2143 # lack metadata (see https://github.com/ytdl-org/youtube-dl/pull/5993).
2144 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
2145 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
2146 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
2147
2148 if width is None:
2149 width = int_or_none(fmt.get('width'))
2150 if height is None:
2151 height = int_or_none(fmt.get('height'))
2152
2153 filesize = int_or_none(url_data.get(
2154 'clen', [None])[0]) or _extract_filesize(url)
2155
2156 quality = url_data.get('quality', [None])[0] or fmt.get('quality')
2157 quality_label = url_data.get('quality_label', [None])[0] or fmt.get('qualityLabel')
2158
2159 tbr = (float_or_none(url_data.get('bitrate', [None])[0], 1000)
2160 or float_or_none(fmt.get('bitrate'), 1000)) if format_id != '43' else None
2161 fps = int_or_none(url_data.get('fps', [None])[0]) or int_or_none(fmt.get('fps'))
2162
2163 more_fields = {
2164 'filesize': filesize,
2165 'tbr': tbr,
2166 'width': width,
2167 'height': height,
2168 'fps': fps,
2169 'format_note': quality_label or quality,
2170 }
2171 for key, value in more_fields.items():
2172 if value:
2173 dct[key] = value
2174 type_ = url_data.get('type', [None])[0] or fmt.get('mimeType')
2175 if type_:
2176 type_split = type_.split(';')
2177 kind_ext = type_split[0].split('/')
2178 if len(kind_ext) == 2:
2179 kind, _ = kind_ext
2180 dct['ext'] = mimetype2ext(type_split[0])
2181 if kind in ('audio', 'video'):
2182 codecs = None
2183 for mobj in re.finditer(
2184 r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
2185 if mobj.group('key') == 'codecs':
2186 codecs = mobj.group('val')
2187 break
2188 if codecs:
2189 dct.update(parse_codecs(codecs))
2190 if dct.get('acodec') == 'none' or dct.get('vcodec') == 'none':
2191 dct['downloader_options'] = {
2192 # Youtube throttles chunks >~10M
2193 'http_chunk_size': 10485760,
2194 }
2195 formats.append(dct)
2196 else:
2197 manifest_url = (
2198 url_or_none(try_get(
2199 player_response,
2200 lambda x: x['streamingData']['hlsManifestUrl'],
2201 compat_str))
2202 or url_or_none(try_get(
2203 video_info, lambda x: x['hlsvp'][0], compat_str)))
2204 if manifest_url:
2205 formats = []
2206 m3u8_formats = self._extract_m3u8_formats(
2207 manifest_url, video_id, 'mp4', fatal=False)
2208 for a_format in m3u8_formats:
2209 itag = self._search_regex(
2210 r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
2211 if itag:
2212 a_format['format_id'] = itag
2213 if itag in self._formats:
2214 dct = self._formats[itag].copy()
2215 dct.update(a_format)
2216 a_format = dct
2217 a_format['player_url'] = player_url
2218 # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
2219 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
2220 formats.append(a_format)
2221 else:
2222 error_message = extract_unavailable_message()
2223 if not error_message:
2224 error_message = clean_html(try_get(
2225 player_response, lambda x: x['playabilityStatus']['reason'],
2226 compat_str))
2227 if not error_message:
2228 error_message = clean_html(
2229 try_get(video_info, lambda x: x['reason'][0], compat_str))
2230 if error_message:
2231 raise ExtractorError(error_message, expected=True)
2232 raise ExtractorError('no conn, hlsvp, hlsManifestUrl or url_encoded_fmt_stream_map information found in video info')
2233
2234 # uploader
2235 video_uploader = try_get(
2236 video_info, lambda x: x['author'][0],
2237 compat_str) or str_or_none(video_details.get('author'))
2238 if video_uploader:
2239 video_uploader = compat_urllib_parse_unquote_plus(video_uploader)
2240 else:
2241 self._downloader.report_warning('unable to extract uploader name')
2242
2243 # uploader_id
2244 video_uploader_id = None
2245 video_uploader_url = None
2246 mobj = re.search(
2247 r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
2248 video_webpage)
2249 if mobj is not None:
2250 video_uploader_id = mobj.group('uploader_id')
2251 video_uploader_url = mobj.group('uploader_url')
2252 else:
2253 owner_profile_url = url_or_none(microformat.get('ownerProfileUrl'))
2254 if owner_profile_url:
2255 video_uploader_id = self._search_regex(
2256 r'(?:user|channel)/([^/]+)', owner_profile_url, 'uploader id',
2257 default=None)
2258 video_uploader_url = owner_profile_url
2259
2260 channel_id = (
2261 str_or_none(video_details.get('channelId'))
2262 or self._html_search_meta(
2263 'channelId', video_webpage, 'channel id', default=None)
2264 or self._search_regex(
2265 r'data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
2266 video_webpage, 'channel id', default=None, group='id'))
2267 channel_url = 'http://www.youtube.com/channel/%s' % channel_id if channel_id else None
2268
2269 thumbnails = []
2270 thumbnails_list = try_get(
2271 video_details, lambda x: x['thumbnail']['thumbnails'], list) or []
2272 for t in thumbnails_list:
2273 if not isinstance(t, dict):
2274 continue
2275 thumbnail_url = url_or_none(t.get('url'))
2276 if not thumbnail_url:
2277 continue
2278 thumbnails.append({
2279 'url': thumbnail_url,
2280 'width': int_or_none(t.get('width')),
2281 'height': int_or_none(t.get('height')),
2282 })
2283
2284 if not thumbnails:
2285 video_thumbnail = None
2286 # We try first to get a high quality image:
2287 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
2288 video_webpage, re.DOTALL)
2289 if m_thumb is not None:
2290 video_thumbnail = m_thumb.group(1)
2291 thumbnail_url = try_get(video_info, lambda x: x['thumbnail_url'][0], compat_str)
2292 if thumbnail_url:
2293 video_thumbnail = compat_urllib_parse_unquote_plus(thumbnail_url)
2294 if video_thumbnail:
2295 thumbnails.append({'url': video_thumbnail})
2296
2297 # upload date
2298 upload_date = self._html_search_meta(
2299 'datePublished', video_webpage, 'upload date', default=None)
2300 if not upload_date:
2301 upload_date = self._search_regex(
2302 [r'(?s)id="eow-date.*?>(.*?)</span>',
2303 r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']'],
2304 video_webpage, 'upload date', default=None)
2305 if not upload_date:
2306 upload_date = microformat.get('publishDate') or microformat.get('uploadDate')
2307 upload_date = unified_strdate(upload_date)
2308
2309 video_license = self._html_search_regex(
2310 r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
2311 video_webpage, 'license', default=None)
2312
2313 m_music = re.search(
2314 r'''(?x)
2315 <h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*
2316 <ul[^>]*>\s*
2317 <li>(?P<title>.+?)
2318 by (?P<creator>.+?)
2319 (?:
2320 \(.+?\)|
2321 <a[^>]*
2322 (?:
2323 \bhref=["\']/red[^>]*>| # drop possible
2324 >\s*Listen ad-free with YouTube Red # YouTube Red ad
2325 )
2326 .*?
2327 )?</li
2328 ''',
2329 video_webpage)
2330 if m_music:
2331 video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
2332 video_creator = clean_html(m_music.group('creator'))
2333 else:
2334 video_alt_title = video_creator = None
2335
2336 def extract_meta(field):
2337 return self._html_search_regex(
2338 r'<h4[^>]+class="title"[^>]*>\s*%s\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li>\s*' % field,
2339 video_webpage, field, default=None)
2340
2341 track = extract_meta('Song')
2342 artist = extract_meta('Artist')
2343 album = extract_meta('Album')
2344
2345 # Youtube Music Auto-generated description
2346 release_date = release_year = None
2347 if video_description:
2348 mobj = re.search(r'(?s)Provided to YouTube by [^\n]+\n+(?P<track>[^Ā·]+)Ā·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?ā„—\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?', video_description)
2349 if mobj:
2350 if not track:
2351 track = mobj.group('track').strip()
2352 if not artist:
2353 artist = mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('Ā·'))
2354 if not album:
2355 album = mobj.group('album'.strip())
2356 release_year = mobj.group('release_year')
2357 release_date = mobj.group('release_date')
2358 if release_date:
2359 release_date = release_date.replace('-', '')
2360 if not release_year:
2361 release_year = int(release_date[:4])
2362 if release_year:
2363 release_year = int(release_year)
2364
2365 m_episode = re.search(
2366 r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*ā€¢\s*E(?P<episode>\d+)</span>',
2367 video_webpage)
2368 if m_episode:
2369 series = unescapeHTML(m_episode.group('series'))
2370 season_number = int(m_episode.group('season'))
2371 episode_number = int(m_episode.group('episode'))
2372 else:
2373 series = season_number = episode_number = None
2374
2375 m_cat_container = self._search_regex(
2376 r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
2377 video_webpage, 'categories', default=None)
2378 category = None
2379 if m_cat_container:
2380 category = self._html_search_regex(
2381 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
2382 default=None)
2383 if not category:
2384 category = try_get(
2385 microformat, lambda x: x['category'], compat_str)
2386 video_categories = None if category is None else [category]
2387
2388 video_tags = [
2389 unescapeHTML(m.group('content'))
2390 for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
2391 if not video_tags:
2392 video_tags = try_get(video_details, lambda x: x['keywords'], list)
2393
2394 def _extract_count(count_name):
2395 return str_to_int(self._search_regex(
2396 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
2397 % re.escape(count_name),
2398 video_webpage, count_name, default=None))
2399
2400 like_count = _extract_count('like')
2401 dislike_count = _extract_count('dislike')
2402
2403 if view_count is None:
2404 view_count = str_to_int(self._search_regex(
2405 r'<[^>]+class=["\']watch-view-count[^>]+>\s*([\d,\s]+)', video_webpage,
2406 'view count', default=None))
2407
2408 average_rating = (
2409 float_or_none(video_details.get('averageRating'))
2410 or try_get(video_info, lambda x: float_or_none(x['avg_rating'][0])))
2411
2412 # subtitles
2413 video_subtitles = self.extract_subtitles(video_id, video_webpage)
2414 automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
2415
2416 video_duration = try_get(
2417 video_info, lambda x: int_or_none(x['length_seconds'][0]))
2418 if not video_duration:
2419 video_duration = int_or_none(video_details.get('lengthSeconds'))
2420 if not video_duration:
2421 video_duration = parse_duration(self._html_search_meta(
2422 'duration', video_webpage, 'video duration'))
2423
2424 # annotations
2425 video_annotations = None
2426 if self._downloader.params.get('writeannotations', False):
2427 xsrf_token = self._search_regex(
2428 r'([\'"])XSRF_TOKEN\1\s*:\s*([\'"])(?P<xsrf_token>[A-Za-z0-9+/=]+)\2',
2429 video_webpage, 'xsrf token', group='xsrf_token', fatal=False)
2430 invideo_url = try_get(
2431 player_response, lambda x: x['annotations'][0]['playerAnnotationsUrlsRenderer']['invideoUrl'], compat_str)
2432 if xsrf_token and invideo_url:
2433 xsrf_field_name = self._search_regex(
2434 r'([\'"])XSRF_FIELD_NAME\1\s*:\s*([\'"])(?P<xsrf_field_name>\w+)\2',
2435 video_webpage, 'xsrf field name',
2436 group='xsrf_field_name', default='session_token')
2437 video_annotations = self._download_webpage(
2438 self._proto_relative_url(invideo_url),
2439 video_id, note='Downloading annotations',
2440 errnote='Unable to download video annotations', fatal=False,
2441 data=urlencode_postdata({xsrf_field_name: xsrf_token}))
2442
2443 chapters = self._extract_chapters(video_webpage, description_original, video_id, video_duration)
2444
2445 # Look for the DASH manifest
2446 if self._downloader.params.get('youtube_include_dash_manifest', True):
2447 dash_mpd_fatal = True
2448 for mpd_url in dash_mpds:
2449 dash_formats = {}
2450 try:
2451 def decrypt_sig(mobj):
2452 s = mobj.group(1)
2453 dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
2454 return '/signature/%s' % dec_s
2455
2456 mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
2457
2458 for df in self._extract_mpd_formats(
2459 mpd_url, video_id, fatal=dash_mpd_fatal,
2460 formats_dict=self._formats):
2461 if not df.get('filesize'):
2462 df['filesize'] = _extract_filesize(df['url'])
2463 # Do not overwrite DASH format found in some previous DASH manifest
2464 if df['format_id'] not in dash_formats:
2465 dash_formats[df['format_id']] = df
2466 # Additional DASH manifests may end up in HTTP Error 403 therefore
2467 # allow them to fail without bug report message if we already have
2468 # some DASH manifest succeeded. This is temporary workaround to reduce
2469 # burst of bug reports until we figure out the reason and whether it
2470 # can be fixed at all.
2471 dash_mpd_fatal = False
2472 except (ExtractorError, KeyError) as e:
2473 self.report_warning(
2474 'Skipping DASH manifest: %r' % e, video_id)
2475 if dash_formats:
2476 # Remove the formats we found through non-DASH, they
2477 # contain less info and it can be wrong, because we use
2478 # fixed values (for example the resolution). See
2479 # https://github.com/ytdl-org/youtube-dl/issues/5774 for an
2480 # example.
2481 formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
2482 formats.extend(dash_formats.values())
2483
2484 # Check for malformed aspect ratio
2485 stretched_m = re.search(
2486 r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
2487 video_webpage)
2488 if stretched_m:
2489 w = float(stretched_m.group('w'))
2490 h = float(stretched_m.group('h'))
2491 # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
2492 # We will only process correct ratios.
2493 if w > 0 and h > 0:
2494 ratio = w / h
2495 for f in formats:
2496 if f.get('vcodec') != 'none':
2497 f['stretched_ratio'] = ratio
2498
2499 if not formats:
2500 if 'reason' in video_info:
2501 if 'The uploader has not made this video available in your country.' in video_info['reason']:
2502 regions_allowed = self._html_search_meta(
2503 'regionsAllowed', video_webpage, default=None)
2504 countries = regions_allowed.split(',') if regions_allowed else None
2505 self.raise_geo_restricted(
2506 msg=video_info['reason'][0], countries=countries)
2507 reason = video_info['reason'][0]
2508 if 'Invalid parameters' in reason:
2509 unavailable_message = extract_unavailable_message()
2510 if unavailable_message:
2511 reason = unavailable_message
2512 raise ExtractorError(
2513 'YouTube said: %s' % reason,
2514 expected=True, video_id=video_id)
2515 if video_info.get('license_info') or try_get(player_response, lambda x: x['streamingData']['licenseInfos']):
2516 raise ExtractorError('This video is DRM protected.', expected=True)
2517
2518 self._sort_formats(formats)
2519
2520 self.mark_watched(video_id, video_info, player_response)
2521
2522 return {
2523 'id': video_id,
2524 'uploader': video_uploader,
2525 'uploader_id': video_uploader_id,
2526 'uploader_url': video_uploader_url,
2527 'channel_id': channel_id,
2528 'channel_url': channel_url,
2529 'upload_date': upload_date,
2530 'license': video_license,
2531 'creator': video_creator or artist,
2532 'title': video_title,
2533 'alt_title': video_alt_title or track,
2534 'thumbnails': thumbnails,
2535 'description': video_description,
2536 'categories': video_categories,
2537 'tags': video_tags,
2538 'subtitles': video_subtitles,
2539 'automatic_captions': automatic_captions,
2540 'duration': video_duration,
2541 'age_limit': 18 if age_gate else 0,
2542 'annotations': video_annotations,
2543 'chapters': chapters,
2544 'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
2545 'view_count': view_count,
2546 'like_count': like_count,
2547 'dislike_count': dislike_count,
2548 'average_rating': average_rating,
2549 'formats': formats,
2550 'is_live': is_live,
2551 'start_time': start_time,
2552 'end_time': end_time,
2553 'series': series,
2554 'season_number': season_number,
2555 'episode_number': episode_number,
2556 'track': track,
2557 'artist': artist,
2558 'album': album,
2559 'release_date': release_date,
2560 'release_year': release_year,
2561 }
2562
2563
2564 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
2565 IE_DESC = 'YouTube.com playlists'
2566 _VALID_URL = r"""(?x)(?:
2567 (?:https?://)?
2568 (?:\w+\.)?
2569 (?:
2570 (?:
2571 youtube(?:kids)?\.com|
2572 invidio\.us
2573 )
2574 /
2575 (?:
2576 (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/(?:videoseries|[0-9A-Za-z_-]{11}))
2577 \? (?:.*?[&;])*? (?:p|a|list)=
2578 | p/
2579 )|
2580 youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
2581 )
2582 (
2583 (?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)?[0-9A-Za-z-_]{10,}
2584 # Top tracks, they can also include dots
2585 |(?:MC)[\w\.]*
2586 )
2587 .*
2588 |
2589 (%(playlist_id)s)
2590 )""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
2591 _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
2592 _VIDEO_RE_TPL = r'href="\s*/watch\?v=%s(?:&amp;(?:[^"]*?index=(?P<index>\d+))?(?:[^>]+>(?P<title>[^<]+))?)?'
2593 _VIDEO_RE = _VIDEO_RE_TPL % r'(?P<id>[0-9A-Za-z_-]{11})'
2594 IE_NAME = 'youtube:playlist'
2595 _TESTS = [{
2596 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2597 'info_dict': {
2598 'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2599 'uploader': 'Sergey M.',
2600 'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2601 'title': 'youtube-dl public playlist',
2602 },
2603 'playlist_count': 1,
2604 }, {
2605 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2606 'info_dict': {
2607 'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2608 'uploader': 'Sergey M.',
2609 'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2610 'title': 'youtube-dl empty playlist',
2611 },
2612 'playlist_count': 0,
2613 }, {
2614 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
2615 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2616 'info_dict': {
2617 'title': '29C3: Not my department',
2618 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2619 'uploader': 'Christiaan008',
2620 'uploader_id': 'ChRiStIaAn008',
2621 },
2622 'playlist_count': 96,
2623 }, {
2624 'note': 'issue #673',
2625 'url': 'PLBB231211A4F62143',
2626 'info_dict': {
2627 'title': '[OLD]Team Fortress 2 (Class-based LP)',
2628 'id': 'PLBB231211A4F62143',
2629 'uploader': 'Wickydoo',
2630 'uploader_id': 'Wickydoo',
2631 },
2632 'playlist_mincount': 26,
2633 }, {
2634 'note': 'Large playlist',
2635 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
2636 'info_dict': {
2637 'title': 'Uploads from Cauchemar',
2638 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
2639 'uploader': 'Cauchemar',
2640 'uploader_id': 'Cauchemar89',
2641 },
2642 'playlist_mincount': 799,
2643 }, {
2644 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2645 'info_dict': {
2646 'title': 'YDL_safe_search',
2647 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2648 },
2649 'playlist_count': 2,
2650 'skip': 'This playlist is private',
2651 }, {
2652 'note': 'embedded',
2653 'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2654 'playlist_count': 4,
2655 'info_dict': {
2656 'title': 'JODA15',
2657 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2658 'uploader': 'milan',
2659 'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
2660 }
2661 }, {
2662 'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2663 'playlist_mincount': 485,
2664 'info_dict': {
2665 'title': '2018 Chinese New Singles (11/6 updated)',
2666 'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2667 'uploader': 'LBK',
2668 'uploader_id': 'sdragonfang',
2669 }
2670 }, {
2671 'note': 'Embedded SWF player',
2672 'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
2673 'playlist_count': 4,
2674 'info_dict': {
2675 'title': 'JODA7',
2676 'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
2677 },
2678 'skip': 'This playlist does not exist',
2679 }, {
2680 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
2681 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
2682 'info_dict': {
2683 'title': 'Uploads from Interstellar Movie',
2684 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
2685 'uploader': 'Interstellar Movie',
2686 'uploader_id': 'InterstellarMovie1',
2687 },
2688 'playlist_mincount': 21,
2689 }, {
2690 # Playlist URL that does not actually serve a playlist
2691 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
2692 'info_dict': {
2693 'id': 'FqZTN594JQw',
2694 'ext': 'webm',
2695 'title': "Smiley's People 01 detective, Adventure Series, Action",
2696 'uploader': 'STREEM',
2697 'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
2698 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
2699 'upload_date': '20150526',
2700 'license': 'Standard YouTube License',
2701 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
2702 'categories': ['People & Blogs'],
2703 'tags': list,
2704 'view_count': int,
2705 'like_count': int,
2706 'dislike_count': int,
2707 },
2708 'params': {
2709 'skip_download': True,
2710 },
2711 'skip': 'This video is not available.',
2712 'add_ie': [YoutubeIE.ie_key()],
2713 }, {
2714 'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
2715 'info_dict': {
2716 'id': 'yeWKywCrFtk',
2717 'ext': 'mp4',
2718 'title': 'Small Scale Baler and Braiding Rugs',
2719 'uploader': 'Backus-Page House Museum',
2720 'uploader_id': 'backuspagemuseum',
2721 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
2722 'upload_date': '20161008',
2723 'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
2724 'categories': ['Nonprofits & Activism'],
2725 'tags': list,
2726 'like_count': int,
2727 'dislike_count': int,
2728 },
2729 'params': {
2730 'noplaylist': True,
2731 'skip_download': True,
2732 },
2733 }, {
2734 # https://github.com/ytdl-org/youtube-dl/issues/21844
2735 'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2736 'info_dict': {
2737 'title': 'Data Analysis with Dr Mike Pound',
2738 'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2739 'uploader_id': 'Computerphile',
2740 'uploader': 'Computerphile',
2741 },
2742 'playlist_mincount': 11,
2743 }, {
2744 'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
2745 'only_matching': True,
2746 }, {
2747 'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
2748 'only_matching': True,
2749 }, {
2750 # music album playlist
2751 'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
2752 'only_matching': True,
2753 }, {
2754 'url': 'https://invidio.us/playlist?list=PLDIoUOhQQPlXr63I_vwF9GD8sAKh77dWU',
2755 'only_matching': True,
2756 }, {
2757 'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
2758 'only_matching': True,
2759 }]
2760
2761 def _real_initialize(self):
2762 self._login()
2763
2764 def extract_videos_from_page(self, page):
2765 ids_in_page = []
2766 titles_in_page = []
2767
2768 for item in re.findall(
2769 r'(<[^>]*\bdata-video-id\s*=\s*["\'][0-9A-Za-z_-]{11}[^>]+>)', page):
2770 attrs = extract_attributes(item)
2771 video_id = attrs['data-video-id']
2772 video_title = unescapeHTML(attrs.get('data-title'))
2773 if video_title:
2774 video_title = video_title.strip()
2775 ids_in_page.append(video_id)
2776 titles_in_page.append(video_title)
2777
2778 # Fallback with old _VIDEO_RE
2779 self.extract_videos_from_page_impl(
2780 self._VIDEO_RE, page, ids_in_page, titles_in_page)
2781
2782 # Relaxed fallbacks
2783 self.extract_videos_from_page_impl(
2784 r'href="\s*/watch\?v\s*=\s*(?P<id>[0-9A-Za-z_-]{11})', page,
2785 ids_in_page, titles_in_page)
2786 self.extract_videos_from_page_impl(
2787 r'data-video-ids\s*=\s*["\'](?P<id>[0-9A-Za-z_-]{11})', page,
2788 ids_in_page, titles_in_page)
2789
2790 return zip(ids_in_page, titles_in_page)
2791
2792 def _extract_mix(self, playlist_id):
2793 # The mixes are generated from a single video
2794 # the id of the playlist is just 'RD' + video_id
2795 ids = []
2796 last_id = playlist_id[-11:]
2797 for n in itertools.count(1):
2798 url = 'https://www.youtube.com/watch?v=%s&list=%s' % (last_id, playlist_id)
2799 webpage = self._download_webpage(
2800 url, playlist_id, 'Downloading page {0} of Youtube mix'.format(n))
2801 new_ids = orderedSet(re.findall(
2802 r'''(?xs)data-video-username=".*?".*?
2803 href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
2804 webpage))
2805 # Fetch new pages until all the videos are repeated, it seems that
2806 # there are always 51 unique videos.
2807 new_ids = [_id for _id in new_ids if _id not in ids]
2808 if not new_ids:
2809 break
2810 ids.extend(new_ids)
2811 last_id = ids[-1]
2812
2813 url_results = self._ids_to_results(ids)
2814
2815 search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
2816 title_span = (
2817 search_title('playlist-title')
2818 or search_title('title long-title')
2819 or search_title('title'))
2820 title = clean_html(title_span)
2821
2822 return self.playlist_result(url_results, playlist_id, title)
2823
2824 def _extract_playlist(self, playlist_id):
2825 url = self._TEMPLATE_URL % playlist_id
2826 page = self._download_webpage(url, playlist_id)
2827
2828 # the yt-alert-message now has tabindex attribute (see https://github.com/ytdl-org/youtube-dl/issues/11604)
2829 for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
2830 match = match.strip()
2831 # Check if the playlist exists or is private
2832 mobj = re.match(r'[^<]*(?:The|This) playlist (?P<reason>does not exist|is private)[^<]*', match)
2833 if mobj:
2834 reason = mobj.group('reason')
2835 message = 'This playlist %s' % reason
2836 if 'private' in reason:
2837 message += ', use --username or --netrc to access it'
2838 message += '.'
2839 raise ExtractorError(message, expected=True)
2840 elif re.match(r'[^<]*Invalid parameters[^<]*', match):
2841 raise ExtractorError(
2842 'Invalid parameters. Maybe URL is incorrect.',
2843 expected=True)
2844 elif re.match(r'[^<]*Choose your language[^<]*', match):
2845 continue
2846 else:
2847 self.report_warning('Youtube gives an alert message: ' + match)
2848
2849 playlist_title = self._html_search_regex(
2850 r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
2851 page, 'title', default=None)
2852
2853 _UPLOADER_BASE = r'class=["\']pl-header-details[^>]+>\s*<li>\s*<a[^>]+\bhref='
2854 uploader = self._html_search_regex(
2855 r'%s["\']/(?:user|channel)/[^>]+>([^<]+)' % _UPLOADER_BASE,
2856 page, 'uploader', default=None)
2857 mobj = re.search(
2858 r'%s(["\'])(?P<path>/(?:user|channel)/(?P<uploader_id>.+?))\1' % _UPLOADER_BASE,
2859 page)
2860 if mobj:
2861 uploader_id = mobj.group('uploader_id')
2862 uploader_url = compat_urlparse.urljoin(url, mobj.group('path'))
2863 else:
2864 uploader_id = uploader_url = None
2865
2866 has_videos = True
2867
2868 if not playlist_title:
2869 try:
2870 # Some playlist URLs don't actually serve a playlist (e.g.
2871 # https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4)
2872 next(self._entries(page, playlist_id))
2873 except StopIteration:
2874 has_videos = False
2875
2876 playlist = self.playlist_result(
2877 self._entries(page, playlist_id), playlist_id, playlist_title)
2878 playlist.update({
2879 'uploader': uploader,
2880 'uploader_id': uploader_id,
2881 'uploader_url': uploader_url,
2882 })
2883
2884 return has_videos, playlist
2885
2886 def _check_download_just_video(self, url, playlist_id):
2887 # Check if it's a video-specific URL
2888 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
2889 video_id = query_dict.get('v', [None])[0] or self._search_regex(
2890 r'(?:(?:^|//)youtu\.be/|youtube\.com/embed/(?!videoseries))([0-9A-Za-z_-]{11})', url,
2891 'video id', default=None)
2892 if video_id:
2893 if self._downloader.params.get('noplaylist'):
2894 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2895 return video_id, self.url_result(video_id, 'Youtube', video_id=video_id)
2896 else:
2897 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
2898 return video_id, None
2899 return None, None
2900
2901 def _real_extract(self, url):
2902 # Extract playlist id
2903 mobj = re.match(self._VALID_URL, url)
2904 if mobj is None:
2905 raise ExtractorError('Invalid URL: %s' % url)
2906 playlist_id = mobj.group(1) or mobj.group(2)
2907
2908 video_id, video = self._check_download_just_video(url, playlist_id)
2909 if video:
2910 return video
2911
2912 if playlist_id.startswith(('RD', 'UL', 'PU')):
2913 # Mixes require a custom extraction process
2914 return self._extract_mix(playlist_id)
2915
2916 has_videos, playlist = self._extract_playlist(playlist_id)
2917 if has_videos or not video_id:
2918 return playlist
2919
2920 # Some playlist URLs don't actually serve a playlist (see
2921 # https://github.com/ytdl-org/youtube-dl/issues/10537).
2922 # Fallback to plain video extraction if there is a video id
2923 # along with playlist id.
2924 return self.url_result(video_id, 'Youtube', video_id=video_id)
2925
2926
2927 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
2928 IE_DESC = 'YouTube.com channels'
2929 _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie|kids)?\.com|(?:www\.)?invidio\.us)/channel/(?P<id>[0-9A-Za-z_-]+)'
2930 _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
2931 _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
2932 IE_NAME = 'youtube:channel'
2933 _TESTS = [{
2934 'note': 'paginated channel',
2935 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
2936 'playlist_mincount': 91,
2937 'info_dict': {
2938 'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
2939 'title': 'Uploads from lex will',
2940 'uploader': 'lex will',
2941 'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2942 }
2943 }, {
2944 'note': 'Age restricted channel',
2945 # from https://www.youtube.com/user/DeusExOfficial
2946 'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
2947 'playlist_mincount': 64,
2948 'info_dict': {
2949 'id': 'UUs0ifCMCm1icqRbqhUINa0w',
2950 'title': 'Uploads from Deus Ex',
2951 'uploader': 'Deus Ex',
2952 'uploader_id': 'DeusExOfficial',
2953 },
2954 }, {
2955 'url': 'https://invidio.us/channel/UC23qupoDRn9YOAVzeoxjOQA',
2956 'only_matching': True,
2957 }, {
2958 'url': 'https://www.youtubekids.com/channel/UCyu8StPfZWapR6rfW_JgqcA',
2959 'only_matching': True,
2960 }]
2961
2962 @classmethod
2963 def suitable(cls, url):
2964 return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
2965 else super(YoutubeChannelIE, cls).suitable(url))
2966
2967 def _build_template_url(self, url, channel_id):
2968 return self._TEMPLATE_URL % channel_id
2969
2970 def _real_extract(self, url):
2971 channel_id = self._match_id(url)
2972
2973 url = self._build_template_url(url, channel_id)
2974
2975 # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
2976 # Workaround by extracting as a playlist if managed to obtain channel playlist URL
2977 # otherwise fallback on channel by page extraction
2978 channel_page = self._download_webpage(
2979 url + '?view=57', channel_id,
2980 'Downloading channel page', fatal=False)
2981 if channel_page is False:
2982 channel_playlist_id = False
2983 else:
2984 channel_playlist_id = self._html_search_meta(
2985 'channelId', channel_page, 'channel id', default=None)
2986 if not channel_playlist_id:
2987 channel_url = self._html_search_meta(
2988 ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
2989 channel_page, 'channel url', default=None)
2990 if channel_url:
2991 channel_playlist_id = self._search_regex(
2992 r'vnd\.youtube://user/([0-9A-Za-z_-]+)',
2993 channel_url, 'channel id', default=None)
2994 if channel_playlist_id and channel_playlist_id.startswith('UC'):
2995 playlist_id = 'UU' + channel_playlist_id[2:]
2996 return self.url_result(
2997 compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
2998
2999 channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
3000 autogenerated = re.search(r'''(?x)
3001 class="[^"]*?(?:
3002 channel-header-autogenerated-label|
3003 yt-channel-title-autogenerated
3004 )[^"]*"''', channel_page) is not None
3005
3006 if autogenerated:
3007 # The videos are contained in a single page
3008 # the ajax pages can't be used, they are empty
3009 entries = [
3010 self.url_result(
3011 video_id, 'Youtube', video_id=video_id,
3012 video_title=video_title)
3013 for video_id, video_title in self.extract_videos_from_page(channel_page)]
3014 return self.playlist_result(entries, channel_id)
3015
3016 try:
3017 next(self._entries(channel_page, channel_id))
3018 except StopIteration:
3019 alert_message = self._html_search_regex(
3020 r'(?s)<div[^>]+class=(["\']).*?\byt-alert-message\b.*?\1[^>]*>(?P<alert>[^<]+)</div>',
3021 channel_page, 'alert', default=None, group='alert')
3022 if alert_message:
3023 raise ExtractorError('Youtube said: %s' % alert_message, expected=True)
3024
3025 return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
3026
3027
3028 class YoutubeUserIE(YoutubeChannelIE):
3029 IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
3030 _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:(?P<user>user|c)/)?(?!(?:attribution_link|watch|results|shared)(?:$|[^a-z_A-Z0-9%-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_%-]+)'
3031 _TEMPLATE_URL = 'https://www.youtube.com/%s/%s/videos'
3032 IE_NAME = 'youtube:user'
3033
3034 _TESTS = [{
3035 'url': 'https://www.youtube.com/user/TheLinuxFoundation',
3036 'playlist_mincount': 320,
3037 'info_dict': {
3038 'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
3039 'title': 'Uploads from The Linux Foundation',
3040 'uploader': 'The Linux Foundation',
3041 'uploader_id': 'TheLinuxFoundation',
3042 }
3043 }, {
3044 # Only available via https://www.youtube.com/c/12minuteathlete/videos
3045 # but not https://www.youtube.com/user/12minuteathlete/videos
3046 'url': 'https://www.youtube.com/c/12minuteathlete/videos',
3047 'playlist_mincount': 249,
3048 'info_dict': {
3049 'id': 'UUVjM-zV6_opMDx7WYxnjZiQ',
3050 'title': 'Uploads from 12 Minute Athlete',
3051 'uploader': '12 Minute Athlete',
3052 'uploader_id': 'the12minuteathlete',
3053 }
3054 }, {
3055 'url': 'ytuser:phihag',
3056 'only_matching': True,
3057 }, {
3058 'url': 'https://www.youtube.com/c/gametrailers',
3059 'only_matching': True,
3060 }, {
3061 'url': 'https://www.youtube.com/c/Pawe%C5%82Zadro%C5%BCniak',
3062 'only_matching': True,
3063 }, {
3064 'url': 'https://www.youtube.com/gametrailers',
3065 'only_matching': True,
3066 }, {
3067 # This channel is not available, geo restricted to JP
3068 'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
3069 'only_matching': True,
3070 }]
3071
3072 @classmethod
3073 def suitable(cls, url):
3074 # Don't return True if the url can be extracted with other youtube
3075 # extractor, the regex would is too permissive and it would match.
3076 other_yt_ies = iter(klass for (name, klass) in globals().items() if name.startswith('Youtube') and name.endswith('IE') and klass is not cls)
3077 if any(ie.suitable(url) for ie in other_yt_ies):
3078 return False
3079 else:
3080 return super(YoutubeUserIE, cls).suitable(url)
3081
3082 def _build_template_url(self, url, channel_id):
3083 mobj = re.match(self._VALID_URL, url)
3084 return self._TEMPLATE_URL % (mobj.group('user') or 'user', mobj.group('id'))
3085
3086
3087 class YoutubeLiveIE(YoutubeBaseInfoExtractor):
3088 IE_DESC = 'YouTube.com live streams'
3089 _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:(?:user|channel|c)/)?(?P<id>[^/]+))/live'
3090 IE_NAME = 'youtube:live'
3091
3092 _TESTS = [{
3093 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
3094 'info_dict': {
3095 'id': 'a48o2S1cPoo',
3096 'ext': 'mp4',
3097 'title': 'The Young Turks - Live Main Show',
3098 'uploader': 'The Young Turks',
3099 'uploader_id': 'TheYoungTurks',
3100 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
3101 'upload_date': '20150715',
3102 'license': 'Standard YouTube License',
3103 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
3104 'categories': ['News & Politics'],
3105 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
3106 'like_count': int,
3107 'dislike_count': int,
3108 },
3109 'params': {
3110 'skip_download': True,
3111 },
3112 }, {
3113 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
3114 'only_matching': True,
3115 }, {
3116 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
3117 'only_matching': True,
3118 }, {
3119 'url': 'https://www.youtube.com/TheYoungTurks/live',
3120 'only_matching': True,
3121 }]
3122
3123 def _real_extract(self, url):
3124 mobj = re.match(self._VALID_URL, url)
3125 channel_id = mobj.group('id')
3126 base_url = mobj.group('base_url')
3127 webpage = self._download_webpage(url, channel_id, fatal=False)
3128 if webpage:
3129 page_type = self._og_search_property(
3130 'type', webpage, 'page type', default='')
3131 video_id = self._html_search_meta(
3132 'videoId', webpage, 'video id', default=None)
3133 if page_type.startswith('video') and video_id and re.match(
3134 r'^[0-9A-Za-z_-]{11}$', video_id):
3135 return self.url_result(video_id, YoutubeIE.ie_key())
3136 return self.url_result(base_url)
3137
3138
3139 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
3140 IE_DESC = 'YouTube.com user/channel playlists'
3141 _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel|c)/(?P<id>[^/]+)/playlists'
3142 IE_NAME = 'youtube:playlists'
3143
3144 _TESTS = [{
3145 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
3146 'playlist_mincount': 4,
3147 'info_dict': {
3148 'id': 'ThirstForScience',
3149 'title': 'ThirstForScience',
3150 },
3151 }, {
3152 # with "Load more" button
3153 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
3154 'playlist_mincount': 70,
3155 'info_dict': {
3156 'id': 'igorkle1',
3157 'title': 'Š˜Š³Š¾Ń€ŃŒ ŠšŠ»ŠµŠ¹Š½ŠµŃ€',
3158 },
3159 }, {
3160 'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
3161 'playlist_mincount': 17,
3162 'info_dict': {
3163 'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
3164 'title': 'Chem Player',
3165 },
3166 'skip': 'Blocked',
3167 }, {
3168 'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
3169 'only_matching': True,
3170 }]
3171
3172
3173 class YoutubeSearchBaseInfoExtractor(YoutubePlaylistBaseInfoExtractor):
3174 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
3175
3176
3177 class YoutubeSearchIE(SearchInfoExtractor, YoutubeSearchBaseInfoExtractor):
3178 IE_DESC = 'YouTube.com searches'
3179 # there doesn't appear to be a real limit, for example if you search for
3180 # 'python' you get more than 8.000.000 results
3181 _MAX_RESULTS = float('inf')
3182 IE_NAME = 'youtube:search'
3183 _SEARCH_KEY = 'ytsearch'
3184 _EXTRA_QUERY_ARGS = {}
3185 _TESTS = []
3186
3187 def _get_n_results(self, query, n):
3188 """Get a specified number of results for a query"""
3189
3190 videos = []
3191 limit = n
3192
3193 url_query = {
3194 'search_query': query.encode('utf-8'),
3195 }
3196 url_query.update(self._EXTRA_QUERY_ARGS)
3197 result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
3198
3199 for pagenum in itertools.count(1):
3200 data = self._download_json(
3201 result_url, video_id='query "%s"' % query,
3202 note='Downloading page %s' % pagenum,
3203 errnote='Unable to download API page',
3204 query={'spf': 'navigate'})
3205 html_content = data[1]['body']['content']
3206
3207 if 'class="search-message' in html_content:
3208 raise ExtractorError(
3209 '[youtube] No video results', expected=True)
3210
3211 new_videos = list(self._process_page(html_content))
3212 videos += new_videos
3213 if not new_videos or len(videos) > limit:
3214 break
3215 next_link = self._html_search_regex(
3216 r'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*<span[^>]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next',
3217 html_content, 'next link', default=None)
3218 if next_link is None:
3219 break
3220 result_url = compat_urlparse.urljoin('https://www.youtube.com/', next_link)
3221
3222 if len(videos) > n:
3223 videos = videos[:n]
3224 return self.playlist_result(videos, query)
3225
3226
3227 class YoutubeSearchDateIE(YoutubeSearchIE):
3228 IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
3229 _SEARCH_KEY = 'ytsearchdate'
3230 IE_DESC = 'YouTube.com searches, newest videos first'
3231 _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
3232
3233
3234 class YoutubeSearchURLIE(YoutubeSearchBaseInfoExtractor):
3235 IE_DESC = 'YouTube.com search URLs'
3236 IE_NAME = 'youtube:search_url'
3237 _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
3238 _TESTS = [{
3239 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
3240 'playlist_mincount': 5,
3241 'info_dict': {
3242 'title': 'youtube-dl test video',
3243 }
3244 }, {
3245 'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
3246 'only_matching': True,
3247 }]
3248
3249 def _real_extract(self, url):
3250 mobj = re.match(self._VALID_URL, url)
3251 query = compat_urllib_parse_unquote_plus(mobj.group('query'))
3252 webpage = self._download_webpage(url, query)
3253 return self.playlist_result(self._process_page(webpage), playlist_title=query)
3254
3255
3256 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
3257 IE_DESC = 'YouTube.com (multi-season) shows'
3258 _VALID_URL = r'https?://(?:www\.)?youtube\.com/show/(?P<id>[^?#]*)'
3259 IE_NAME = 'youtube:show'
3260 _TESTS = [{
3261 'url': 'https://www.youtube.com/show/airdisasters',
3262 'playlist_mincount': 5,
3263 'info_dict': {
3264 'id': 'airdisasters',
3265 'title': 'Air Disasters',
3266 }
3267 }]
3268
3269 def _real_extract(self, url):
3270 playlist_id = self._match_id(url)
3271 return super(YoutubeShowIE, self)._real_extract(
3272 'https://www.youtube.com/show/%s/playlists' % playlist_id)
3273
3274
3275 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
3276 """
3277 Base class for feed extractors
3278 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
3279 """
3280 _LOGIN_REQUIRED = True
3281
3282 @property
3283 def IE_NAME(self):
3284 return 'youtube:%s' % self._FEED_NAME
3285
3286 def _real_initialize(self):
3287 self._login()
3288
3289 def _entries(self, page):
3290 # The extraction process is the same as for playlists, but the regex
3291 # for the video ids doesn't contain an index
3292 ids = []
3293 more_widget_html = content_html = page
3294 for page_num in itertools.count(1):
3295 matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
3296
3297 # 'recommended' feed has infinite 'load more' and each new portion spins
3298 # the same videos in (sometimes) slightly different order, so we'll check
3299 # for unicity and break when portion has no new videos
3300 new_ids = list(filter(lambda video_id: video_id not in ids, orderedSet(matches)))
3301 if not new_ids:
3302 break
3303
3304 ids.extend(new_ids)
3305
3306 for entry in self._ids_to_results(new_ids):
3307 yield entry
3308
3309 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
3310 if not mobj:
3311 break
3312
3313 more = self._download_json(
3314 'https://www.youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
3315 'Downloading page #%s' % page_num,
3316 transform_source=uppercase_escape,
3317 headers=self._YOUTUBE_CLIENT_HEADERS)
3318 content_html = more['content_html']
3319 more_widget_html = more['load_more_widget_html']
3320
3321 def _real_extract(self, url):
3322 page = self._download_webpage(
3323 'https://www.youtube.com/feed/%s' % self._FEED_NAME,
3324 self._PLAYLIST_TITLE)
3325 return self.playlist_result(
3326 self._entries(page), playlist_title=self._PLAYLIST_TITLE)
3327
3328
3329 class YoutubeWatchLaterIE(YoutubePlaylistIE):
3330 IE_NAME = 'youtube:watchlater'
3331 IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
3332 _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
3333
3334 _TESTS = [{
3335 'url': 'https://www.youtube.com/playlist?list=WL',
3336 'only_matching': True,
3337 }, {
3338 'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
3339 'only_matching': True,
3340 }]
3341
3342 def _real_extract(self, url):
3343 _, video = self._check_download_just_video(url, 'WL')
3344 if video:
3345 return video
3346 _, playlist = self._extract_playlist('WL')
3347 return playlist
3348
3349
3350 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
3351 IE_NAME = 'youtube:favorites'
3352 IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
3353 _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
3354 _LOGIN_REQUIRED = True
3355
3356 def _real_extract(self, url):
3357 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
3358 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
3359 return self.url_result(playlist_id, 'YoutubePlaylist')
3360
3361
3362 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
3363 IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
3364 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/recommended|:ytrec(?:ommended)?'
3365 _FEED_NAME = 'recommended'
3366 _PLAYLIST_TITLE = 'Youtube Recommended videos'
3367
3368
3369 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
3370 IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
3371 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
3372 _FEED_NAME = 'subscriptions'
3373 _PLAYLIST_TITLE = 'Youtube Subscriptions'
3374
3375
3376 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
3377 IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
3378 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
3379 _FEED_NAME = 'history'
3380 _PLAYLIST_TITLE = 'Youtube History'
3381
3382
3383 class YoutubeTruncatedURLIE(InfoExtractor):
3384 IE_NAME = 'youtube:truncated_url'
3385 IE_DESC = False # Do not list
3386 _VALID_URL = r'''(?x)
3387 (?:https?://)?
3388 (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
3389 (?:watch\?(?:
3390 feature=[a-z_]+|
3391 annotation_id=annotation_[^&]+|
3392 x-yt-cl=[0-9]+|
3393 hl=[^&]*|
3394 t=[0-9]+
3395 )?
3396 |
3397 attribution_link\?a=[^&]+
3398 )
3399 $
3400 '''
3401
3402 _TESTS = [{
3403 'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
3404 'only_matching': True,
3405 }, {
3406 'url': 'https://www.youtube.com/watch?',
3407 'only_matching': True,
3408 }, {
3409 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
3410 'only_matching': True,
3411 }, {
3412 'url': 'https://www.youtube.com/watch?feature=foo',
3413 'only_matching': True,
3414 }, {
3415 'url': 'https://www.youtube.com/watch?hl=en-GB',
3416 'only_matching': True,
3417 }, {
3418 'url': 'https://www.youtube.com/watch?t=2372',
3419 'only_matching': True,
3420 }]
3421
3422 def _real_extract(self, url):
3423 raise ExtractorError(
3424 'Did you forget to quote the URL? Remember that & is a meta '
3425 'character in most shells, so you want to put the URL in quotes, '
3426 'like youtube-dl '
3427 '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
3428 ' or simply youtube-dl BaW_jenozKc .',
3429 expected=True)
3430
3431
3432 class YoutubeTruncatedIDIE(InfoExtractor):
3433 IE_NAME = 'youtube:truncated_id'
3434 IE_DESC = False # Do not list
3435 _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
3436
3437 _TESTS = [{
3438 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
3439 'only_matching': True,
3440 }]
3441
3442 def _real_extract(self, url):
3443 video_id = self._match_id(url)
3444 raise ExtractorError(
3445 'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
3446 expected=True)