]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/youtube.py
3f49f3889e6bcde1bd0f641f9f421caf8d560b1c
[youtubedl] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 from __future__ import unicode_literals
4
5
6 import itertools
7 import json
8 import os.path
9 import random
10 import re
11 import time
12 import traceback
13
14 from .common import InfoExtractor, SearchInfoExtractor
15 from ..jsinterp import JSInterpreter
16 from ..swfinterp import SWFInterpreter
17 from ..compat import (
18 compat_chr,
19 compat_kwargs,
20 compat_parse_qs,
21 compat_urllib_parse_unquote,
22 compat_urllib_parse_unquote_plus,
23 compat_urllib_parse_urlencode,
24 compat_urllib_parse_urlparse,
25 compat_urlparse,
26 compat_str,
27 )
28 from ..utils import (
29 clean_html,
30 error_to_compat_str,
31 ExtractorError,
32 float_or_none,
33 get_element_by_attribute,
34 get_element_by_id,
35 int_or_none,
36 mimetype2ext,
37 orderedSet,
38 parse_codecs,
39 parse_duration,
40 qualities,
41 remove_quotes,
42 remove_start,
43 smuggle_url,
44 str_or_none,
45 str_to_int,
46 try_get,
47 unescapeHTML,
48 unified_strdate,
49 unsmuggle_url,
50 uppercase_escape,
51 urlencode_postdata,
52 )
53
54
55 class YoutubeBaseInfoExtractor(InfoExtractor):
56 """Provide base functions for Youtube extractors"""
57 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
58 _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
59
60 _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
61 _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
62 _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
63
64 _NETRC_MACHINE = 'youtube'
65 # If True it will raise an error if no login info is provided
66 _LOGIN_REQUIRED = False
67
68 _PLAYLIST_ID_RE = r'(?:PL|LL|EC|UU|FL|RD|UL|TL|OLAK5uy_)[0-9A-Za-z-_]{10,}'
69
70 def _set_language(self):
71 self._set_cookie(
72 '.youtube.com', 'PREF', 'f1=50000000&hl=en',
73 # YouTube sets the expire time to about two months
74 expire_time=time.time() + 2 * 30 * 24 * 3600)
75
76 def _ids_to_results(self, ids):
77 return [
78 self.url_result(vid_id, 'Youtube', video_id=vid_id)
79 for vid_id in ids]
80
81 def _login(self):
82 """
83 Attempt to log in to YouTube.
84 True is returned if successful or skipped.
85 False is returned if login failed.
86
87 If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
88 """
89 username, password = self._get_login_info()
90 # No authentication to be performed
91 if username is None:
92 if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
93 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
94 return True
95
96 login_page = self._download_webpage(
97 self._LOGIN_URL, None,
98 note='Downloading login page',
99 errnote='unable to fetch login page', fatal=False)
100 if login_page is False:
101 return
102
103 login_form = self._hidden_inputs(login_page)
104
105 def req(url, f_req, note, errnote):
106 data = login_form.copy()
107 data.update({
108 'pstMsg': 1,
109 'checkConnection': 'youtube',
110 'checkedDomains': 'youtube',
111 'hl': 'en',
112 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
113 'f.req': json.dumps(f_req),
114 'flowName': 'GlifWebSignIn',
115 'flowEntry': 'ServiceLogin',
116 })
117 return self._download_json(
118 url, None, note=note, errnote=errnote,
119 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
120 fatal=False,
121 data=urlencode_postdata(data), headers={
122 'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
123 'Google-Accounts-XSRF': 1,
124 })
125
126 def warn(message):
127 self._downloader.report_warning(message)
128
129 lookup_req = [
130 username,
131 None, [], None, 'US', None, None, 2, False, True,
132 [
133 None, None,
134 [2, 1, None, 1,
135 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
136 None, [], 4],
137 1, [None, None, []], None, None, None, True
138 ],
139 username,
140 ]
141
142 lookup_results = req(
143 self._LOOKUP_URL, lookup_req,
144 'Looking up account info', 'Unable to look up account info')
145
146 if lookup_results is False:
147 return False
148
149 user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
150 if not user_hash:
151 warn('Unable to extract user hash')
152 return False
153
154 challenge_req = [
155 user_hash,
156 None, 1, None, [1, None, None, None, [password, None, True]],
157 [
158 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
159 1, [None, None, []], None, None, None, True
160 ]]
161
162 challenge_results = req(
163 self._CHALLENGE_URL, challenge_req,
164 'Logging in', 'Unable to log in')
165
166 if challenge_results is False:
167 return
168
169 login_res = try_get(challenge_results, lambda x: x[0][5], list)
170 if login_res:
171 login_msg = try_get(login_res, lambda x: x[5], compat_str)
172 warn(
173 'Unable to login: %s' % 'Invalid password'
174 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
175 return False
176
177 res = try_get(challenge_results, lambda x: x[0][-1], list)
178 if not res:
179 warn('Unable to extract result entry')
180 return False
181
182 login_challenge = try_get(res, lambda x: x[0][0], list)
183 if login_challenge:
184 challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
185 if challenge_str == 'TWO_STEP_VERIFICATION':
186 # SEND_SUCCESS - TFA code has been successfully sent to phone
187 # QUOTA_EXCEEDED - reached the limit of TFA codes
188 status = try_get(login_challenge, lambda x: x[5], compat_str)
189 if status == 'QUOTA_EXCEEDED':
190 warn('Exceeded the limit of TFA codes, try later')
191 return False
192
193 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
194 if not tl:
195 warn('Unable to extract TL')
196 return False
197
198 tfa_code = self._get_tfa_info('2-step verification code')
199
200 if not tfa_code:
201 warn(
202 'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
203 '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
204 return False
205
206 tfa_code = remove_start(tfa_code, 'G-')
207
208 tfa_req = [
209 user_hash, None, 2, None,
210 [
211 9, None, None, None, None, None, None, None,
212 [None, tfa_code, True, 2]
213 ]]
214
215 tfa_results = req(
216 self._TFA_URL.format(tl), tfa_req,
217 'Submitting TFA code', 'Unable to submit TFA code')
218
219 if tfa_results is False:
220 return False
221
222 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
223 if tfa_res:
224 tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
225 warn(
226 'Unable to finish TFA: %s' % 'Invalid TFA code'
227 if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
228 return False
229
230 check_cookie_url = try_get(
231 tfa_results, lambda x: x[0][-1][2], compat_str)
232 else:
233 CHALLENGES = {
234 'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
235 'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
236 'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
237 }
238 challenge = CHALLENGES.get(
239 challenge_str,
240 '%s returned error %s.' % (self.IE_NAME, challenge_str))
241 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
242 return False
243 else:
244 check_cookie_url = try_get(res, lambda x: x[2], compat_str)
245
246 if not check_cookie_url:
247 warn('Unable to extract CheckCookie URL')
248 return False
249
250 check_cookie_results = self._download_webpage(
251 check_cookie_url, None, 'Checking cookie', fatal=False)
252
253 if check_cookie_results is False:
254 return False
255
256 if 'https://myaccount.google.com/' not in check_cookie_results:
257 warn('Unable to log in')
258 return False
259
260 return True
261
262 def _download_webpage_handle(self, *args, **kwargs):
263 query = kwargs.get('query', {}).copy()
264 query['disable_polymer'] = 'true'
265 kwargs['query'] = query
266 return super(YoutubeBaseInfoExtractor, self)._download_webpage_handle(
267 *args, **compat_kwargs(kwargs))
268
269 def _real_initialize(self):
270 if self._downloader is None:
271 return
272 self._set_language()
273 if not self._login():
274 return
275
276
277 class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
278 # Extract entries from page with "Load more" button
279 def _entries(self, page, playlist_id):
280 more_widget_html = content_html = page
281 for page_num in itertools.count(1):
282 for entry in self._process_page(content_html):
283 yield entry
284
285 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
286 if not mobj:
287 break
288
289 more = self._download_json(
290 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
291 'Downloading page #%s' % page_num,
292 transform_source=uppercase_escape)
293 content_html = more['content_html']
294 if not content_html.strip():
295 # Some webpages show a "Load more" button but they don't
296 # have more videos
297 break
298 more_widget_html = more['load_more_widget_html']
299
300
301 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
302 def _process_page(self, content):
303 for video_id, video_title in self.extract_videos_from_page(content):
304 yield self.url_result(video_id, 'Youtube', video_id, video_title)
305
306 def extract_videos_from_page(self, page):
307 ids_in_page = []
308 titles_in_page = []
309 for mobj in re.finditer(self._VIDEO_RE, page):
310 # The link with index 0 is not the first video of the playlist (not sure if still actual)
311 if 'index' in mobj.groupdict() and mobj.group('id') == '0':
312 continue
313 video_id = mobj.group('id')
314 video_title = unescapeHTML(mobj.group('title'))
315 if video_title:
316 video_title = video_title.strip()
317 try:
318 idx = ids_in_page.index(video_id)
319 if video_title and not titles_in_page[idx]:
320 titles_in_page[idx] = video_title
321 except ValueError:
322 ids_in_page.append(video_id)
323 titles_in_page.append(video_title)
324 return zip(ids_in_page, titles_in_page)
325
326
327 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
328 def _process_page(self, content):
329 for playlist_id in orderedSet(re.findall(
330 r'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
331 content)):
332 yield self.url_result(
333 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
334
335 def _real_extract(self, url):
336 playlist_id = self._match_id(url)
337 webpage = self._download_webpage(url, playlist_id)
338 title = self._og_search_title(webpage, fatal=False)
339 return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
340
341
342 class YoutubeIE(YoutubeBaseInfoExtractor):
343 IE_DESC = 'YouTube.com'
344 _VALID_URL = r"""(?x)^
345 (
346 (?:https?://|//) # http(s):// or protocol-independent URL
347 (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
348 (?:www\.)?deturl\.com/www\.youtube\.com/|
349 (?:www\.)?pwnyoutube\.com/|
350 (?:www\.)?hooktube\.com/|
351 (?:www\.)?yourepeat\.com/|
352 tube\.majestyc\.net/|
353 (?:www\.)?invidio\.us/|
354 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
355 (?:.*?\#/)? # handle anchor (#/) redirect urls
356 (?: # the various things that can precede the ID:
357 (?:(?:v|embed|e)/(?!videoseries)) # v/ or embed/ or e/
358 |(?: # or the v= param in all its forms
359 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
360 (?:\?|\#!?) # the params delimiter ? or # or #!
361 (?:.*?[&;])?? # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
362 v=
363 )
364 ))
365 |(?:
366 youtu\.be| # just youtu.be/xxxx
367 vid\.plus| # or vid.plus/xxxx
368 zwearz\.com/watch| # or zwearz.com/watch/xxxx
369 )/
370 |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
371 )
372 )? # all until now is optional -> you can pass the naked ID
373 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
374 (?!.*?\blist=
375 (?:
376 %(playlist_id)s| # combined list/video URLs are handled by the playlist IE
377 WL # WL are handled by the watch later IE
378 )
379 )
380 (?(1).+)? # if we found the ID, everything can follow
381 $""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
382 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
383 _formats = {
384 '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
385 '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
386 '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
387 '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
388 '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
389 '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
390 '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
391 '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
392 # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
393 '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
394 '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
395 '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
396 '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
397 '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
398 '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
399 '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
400 '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
401 '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
402
403
404 # 3D videos
405 '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
406 '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
407 '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
408 '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
409 '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
410 '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
411 '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
412
413 # Apple HTTP Live Streaming
414 '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
415 '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
416 '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
417 '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
418 '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
419 '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
420 '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
421 '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
422
423 # DASH mp4 video
424 '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
425 '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
426 '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
427 '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
428 '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
429 '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'}, # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
430 '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
431 '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
432 '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
433 '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
434 '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
435 '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
436
437 # Dash mp4 audio
438 '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
439 '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
440 '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
441 '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
442 '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
443 '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
444 '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
445
446 # Dash webm
447 '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
448 '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
449 '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
450 '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
451 '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
452 '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
453 '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
454 '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
455 '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
456 '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
457 '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
458 '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
459 '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
460 '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
461 '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
462 # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
463 '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
464 '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
465 '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
466 '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
467 '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
468 '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
469
470 # Dash webm audio
471 '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
472 '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
473
474 # Dash webm audio with opus inside
475 '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
476 '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
477 '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
478
479 # RTMP (unnamed)
480 '_rtmp': {'protocol': 'rtmp'},
481 }
482 _SUBTITLE_FORMATS = ('ttml', 'vtt')
483
484 _GEO_BYPASS = False
485
486 IE_NAME = 'youtube'
487 _TESTS = [
488 {
489 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
490 'info_dict': {
491 'id': 'BaW_jenozKc',
492 'ext': 'mp4',
493 'title': 'youtube-dl test video "\'/\\Ƥā†­š•',
494 'uploader': 'Philipp Hagemeister',
495 'uploader_id': 'phihag',
496 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
497 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
498 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
499 'upload_date': '20121002',
500 'license': 'Standard YouTube License',
501 'description': 'test chars: "\'/\\Ƥā†­š•\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
502 'categories': ['Science & Technology'],
503 'tags': ['youtube-dl'],
504 'duration': 10,
505 'view_count': int,
506 'like_count': int,
507 'dislike_count': int,
508 'start_time': 1,
509 'end_time': 9,
510 }
511 },
512 {
513 'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY',
514 'note': 'Test generic use_cipher_signature video (#897)',
515 'info_dict': {
516 'id': 'UxxajLWwzqY',
517 'ext': 'mp4',
518 'upload_date': '20120506',
519 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
520 'alt_title': 'I Love It (feat. Charli XCX)',
521 'description': 'md5:f3ceb5ef83a08d95b9d146f973157cc8',
522 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
523 'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
524 'iconic ep', 'iconic', 'love', 'it'],
525 'duration': 180,
526 'uploader': 'Icona Pop',
527 'uploader_id': 'IconaPop',
528 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop',
529 'license': 'Standard YouTube License',
530 'creator': 'Icona Pop',
531 'track': 'I Love It (feat. Charli XCX)',
532 'artist': 'Icona Pop',
533 }
534 },
535 {
536 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
537 'note': 'Test VEVO video with age protection (#956)',
538 'info_dict': {
539 'id': '07FYdnEawAQ',
540 'ext': 'mp4',
541 'upload_date': '20130703',
542 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
543 'alt_title': 'Tunnel Vision',
544 'description': 'md5:64249768eec3bc4276236606ea996373',
545 'duration': 419,
546 'uploader': 'justintimberlakeVEVO',
547 'uploader_id': 'justintimberlakeVEVO',
548 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
549 'license': 'Standard YouTube License',
550 'creator': 'Justin Timberlake',
551 'track': 'Tunnel Vision',
552 'artist': 'Justin Timberlake',
553 'age_limit': 18,
554 }
555 },
556 {
557 'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
558 'note': 'Embed-only video (#1746)',
559 'info_dict': {
560 'id': 'yZIXLfi8CZQ',
561 'ext': 'mp4',
562 'upload_date': '20120608',
563 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
564 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
565 'uploader': 'SET India',
566 'uploader_id': 'setindia',
567 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
568 'license': 'Standard YouTube License',
569 'age_limit': 18,
570 }
571 },
572 {
573 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
574 'note': 'Use the first video ID in the URL',
575 'info_dict': {
576 'id': 'BaW_jenozKc',
577 'ext': 'mp4',
578 'title': 'youtube-dl test video "\'/\\Ƥā†­š•',
579 'uploader': 'Philipp Hagemeister',
580 'uploader_id': 'phihag',
581 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
582 'upload_date': '20121002',
583 'license': 'Standard YouTube License',
584 'description': 'test chars: "\'/\\Ƥā†­š•\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
585 'categories': ['Science & Technology'],
586 'tags': ['youtube-dl'],
587 'duration': 10,
588 'view_count': int,
589 'like_count': int,
590 'dislike_count': int,
591 },
592 'params': {
593 'skip_download': True,
594 },
595 },
596 {
597 'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
598 'note': '256k DASH audio (format 141) via DASH manifest',
599 'info_dict': {
600 'id': 'a9LDPn-MO4I',
601 'ext': 'm4a',
602 'upload_date': '20121002',
603 'uploader_id': '8KVIDEO',
604 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
605 'description': '',
606 'uploader': '8KVIDEO',
607 'license': 'Standard YouTube License',
608 'title': 'UHDTV TEST 8K VIDEO.mp4'
609 },
610 'params': {
611 'youtube_include_dash_manifest': True,
612 'format': '141',
613 },
614 'skip': 'format 141 not served anymore',
615 },
616 # DASH manifest with encrypted signature
617 {
618 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
619 'info_dict': {
620 'id': 'IB3lcPjvWLA',
621 'ext': 'm4a',
622 'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
623 'description': 'md5:1900ed86ee514927b9e00fbead6969a5',
624 'duration': 244,
625 'uploader': 'AfrojackVEVO',
626 'uploader_id': 'AfrojackVEVO',
627 'upload_date': '20131011',
628 'license': 'Standard YouTube License',
629 },
630 'params': {
631 'youtube_include_dash_manifest': True,
632 'format': '141/bestaudio[ext=m4a]',
633 },
634 },
635 # JS player signature function name containing $
636 {
637 'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
638 'info_dict': {
639 'id': 'nfWlot6h_JM',
640 'ext': 'm4a',
641 'title': 'Taylor Swift - Shake It Off',
642 'alt_title': 'Shake It Off',
643 'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
644 'duration': 242,
645 'uploader': 'TaylorSwiftVEVO',
646 'uploader_id': 'TaylorSwiftVEVO',
647 'upload_date': '20140818',
648 'license': 'Standard YouTube License',
649 'creator': 'Taylor Swift',
650 },
651 'params': {
652 'youtube_include_dash_manifest': True,
653 'format': '141/bestaudio[ext=m4a]',
654 },
655 },
656 # Controversy video
657 {
658 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
659 'info_dict': {
660 'id': 'T4XJQO3qol8',
661 'ext': 'mp4',
662 'duration': 219,
663 'upload_date': '20100909',
664 'uploader': 'TJ Kirk',
665 'uploader_id': 'TheAmazingAtheist',
666 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
667 'license': 'Standard YouTube License',
668 'title': 'Burning Everyone\'s Koran',
669 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
670 }
671 },
672 # Normal age-gate video (No vevo, embed allowed)
673 {
674 'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
675 'info_dict': {
676 'id': 'HtVdAasjOgU',
677 'ext': 'mp4',
678 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
679 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
680 'duration': 142,
681 'uploader': 'The Witcher',
682 'uploader_id': 'WitcherGame',
683 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
684 'upload_date': '20140605',
685 'license': 'Standard YouTube License',
686 'age_limit': 18,
687 },
688 },
689 # Age-gate video with encrypted signature
690 {
691 'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
692 'info_dict': {
693 'id': '6kLq3WMV1nU',
694 'ext': 'webm',
695 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
696 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
697 'duration': 246,
698 'uploader': 'LloydVEVO',
699 'uploader_id': 'LloydVEVO',
700 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
701 'upload_date': '20110629',
702 'license': 'Standard YouTube License',
703 'age_limit': 18,
704 },
705 },
706 # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
707 # YouTube Red ad is not captured for creator
708 {
709 'url': '__2ABJjxzNo',
710 'info_dict': {
711 'id': '__2ABJjxzNo',
712 'ext': 'mp4',
713 'duration': 266,
714 'upload_date': '20100430',
715 'uploader_id': 'deadmau5',
716 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
717 'creator': 'deadmau5',
718 'description': 'md5:12c56784b8032162bb936a5f76d55360',
719 'uploader': 'deadmau5',
720 'license': 'Standard YouTube License',
721 'title': 'Deadmau5 - Some Chords (HD)',
722 'alt_title': 'Some Chords',
723 },
724 'expected_warnings': [
725 'DASH manifest missing',
726 ]
727 },
728 # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
729 {
730 'url': 'lqQg6PlCWgI',
731 'info_dict': {
732 'id': 'lqQg6PlCWgI',
733 'ext': 'mp4',
734 'duration': 6085,
735 'upload_date': '20150827',
736 'uploader_id': 'olympic',
737 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
738 'license': 'Standard YouTube License',
739 'description': 'HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
740 'uploader': 'Olympic',
741 'title': 'Hockey - Women - GER-AUS - London 2012 Olympic Games',
742 },
743 'params': {
744 'skip_download': 'requires avconv',
745 }
746 },
747 # Non-square pixels
748 {
749 'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
750 'info_dict': {
751 'id': '_b-2C3KPAM0',
752 'ext': 'mp4',
753 'stretched_ratio': 16 / 9.,
754 'duration': 85,
755 'upload_date': '20110310',
756 'uploader_id': 'AllenMeow',
757 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
758 'description': 'made by Wacom from Korea | 字幕&åŠ ę²¹ę·»é†‹ by TY\'s Allen | ę„Ÿč¬heylisa00cavey1001同å­øē†±ęƒ…ęä¾›ę¢—åŠēæ»č­Æ',
759 'uploader': 'å­«į„‹į„…',
760 'license': 'Standard YouTube License',
761 'title': '[A-made] č®Šę…‹å¦å­—å¹•ē‰ˆ å¤Ŗ妍 ęˆ‘å°±ę˜Æ這ęØ£ēš„äŗŗ',
762 },
763 },
764 # url_encoded_fmt_stream_map is empty string
765 {
766 'url': 'qEJwOuvDf7I',
767 'info_dict': {
768 'id': 'qEJwOuvDf7I',
769 'ext': 'webm',
770 'title': 'ŠžŠ±ŃŃƒŠ¶Š“ŠµŠ½ŠøŠµ суŠ“ŠµŠ±Š½Š¾Š¹ ŠæрŠ°ŠŗтŠøŠŗŠø ŠæŠ¾ Š²Ń‹Š±Š¾Ń€Š°Š¼ 14 сŠµŠ½Ń‚яŠ±Ń€Ń 2014 Š³Š¾Š“Š° Š² Š”Š°Š½Šŗт-ŠŸŠµŃ‚ŠµŃ€Š±ŃƒŃ€Š³Šµ',
771 'description': '',
772 'upload_date': '20150404',
773 'uploader_id': 'spbelect',
774 'uploader': 'ŠŠ°Š±Š»ŃŽŠ“Š°Ń‚ŠµŠ»Šø ŠŸŠµŃ‚ŠµŃ€Š±ŃƒŃ€Š³Š°',
775 },
776 'params': {
777 'skip_download': 'requires avconv',
778 },
779 'skip': 'This live event has ended.',
780 },
781 # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
782 {
783 'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
784 'info_dict': {
785 'id': 'FIl7x6_3R5Y',
786 'ext': 'webm',
787 'title': 'md5:7b81415841e02ecd4313668cde88737a',
788 'description': 'md5:116377fd2963b81ec4ce64b542173306',
789 'duration': 220,
790 'upload_date': '20150625',
791 'uploader_id': 'dorappi2000',
792 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
793 'uploader': 'dorappi2000',
794 'license': 'Standard YouTube License',
795 'formats': 'mincount:31',
796 },
797 'skip': 'not actual anymore',
798 },
799 # DASH manifest with segment_list
800 {
801 'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
802 'md5': '8ce563a1d667b599d21064e982ab9e31',
803 'info_dict': {
804 'id': 'CsmdDsKjzN8',
805 'ext': 'mp4',
806 'upload_date': '20150501', # According to '<meta itemprop="datePublished"', but in other places it's 20150510
807 'uploader': 'Airtek',
808 'description': 'RetransmisiĆ³n en directo de la XVIII media maratĆ³n de Zaragoza.',
809 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
810 'license': 'Standard YouTube License',
811 'title': 'RetransmisiĆ³n XVIII Media maratĆ³n Zaragoza 2015',
812 },
813 'params': {
814 'youtube_include_dash_manifest': True,
815 'format': '135', # bestvideo
816 },
817 'skip': 'This live event has ended.',
818 },
819 {
820 # Multifeed videos (multiple cameras), URL is for Main Camera
821 'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
822 'info_dict': {
823 'id': 'jqWvoWXjCVs',
824 'title': 'teamPGP: Rocket League Noob Stream',
825 'description': 'md5:dc7872fb300e143831327f1bae3af010',
826 },
827 'playlist': [{
828 'info_dict': {
829 'id': 'jqWvoWXjCVs',
830 'ext': 'mp4',
831 'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
832 'description': 'md5:dc7872fb300e143831327f1bae3af010',
833 'duration': 7335,
834 'upload_date': '20150721',
835 'uploader': 'Beer Games Beer',
836 'uploader_id': 'beergamesbeer',
837 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
838 'license': 'Standard YouTube License',
839 },
840 }, {
841 'info_dict': {
842 'id': '6h8e8xoXJzg',
843 'ext': 'mp4',
844 'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
845 'description': 'md5:dc7872fb300e143831327f1bae3af010',
846 'duration': 7337,
847 'upload_date': '20150721',
848 'uploader': 'Beer Games Beer',
849 'uploader_id': 'beergamesbeer',
850 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
851 'license': 'Standard YouTube License',
852 },
853 }, {
854 'info_dict': {
855 'id': 'PUOgX5z9xZw',
856 'ext': 'mp4',
857 'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
858 'description': 'md5:dc7872fb300e143831327f1bae3af010',
859 'duration': 7337,
860 'upload_date': '20150721',
861 'uploader': 'Beer Games Beer',
862 'uploader_id': 'beergamesbeer',
863 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
864 'license': 'Standard YouTube License',
865 },
866 }, {
867 'info_dict': {
868 'id': 'teuwxikvS5k',
869 'ext': 'mp4',
870 'title': 'teamPGP: Rocket League Noob Stream (zim)',
871 'description': 'md5:dc7872fb300e143831327f1bae3af010',
872 'duration': 7334,
873 'upload_date': '20150721',
874 'uploader': 'Beer Games Beer',
875 'uploader_id': 'beergamesbeer',
876 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
877 'license': 'Standard YouTube License',
878 },
879 }],
880 'params': {
881 'skip_download': True,
882 },
883 },
884 {
885 # Multifeed video with comma in title (see https://github.com/rg3/youtube-dl/issues/8536)
886 'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
887 'info_dict': {
888 'id': 'gVfLd0zydlo',
889 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
890 },
891 'playlist_count': 2,
892 'skip': 'Not multifeed anymore',
893 },
894 {
895 'url': 'https://vid.plus/FlRa-iH7PGw',
896 'only_matching': True,
897 },
898 {
899 'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
900 'only_matching': True,
901 },
902 {
903 # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
904 # Also tests cut-off URL expansion in video description (see
905 # https://github.com/rg3/youtube-dl/issues/1892,
906 # https://github.com/rg3/youtube-dl/issues/8164)
907 'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
908 'info_dict': {
909 'id': 'lsguqyKfVQg',
910 'ext': 'mp4',
911 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
912 'alt_title': 'Dark Walk - Position Music',
913 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
914 'duration': 133,
915 'upload_date': '20151119',
916 'uploader_id': 'IronSoulElf',
917 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
918 'uploader': 'IronSoulElf',
919 'license': 'Standard YouTube License',
920 'creator': 'Todd Haberman, Daniel Law Heath and Aaron Kaplan',
921 'track': 'Dark Walk - Position Music',
922 'artist': 'Todd Haberman, Daniel Law Heath and Aaron Kaplan',
923 },
924 'params': {
925 'skip_download': True,
926 },
927 },
928 {
929 # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
930 'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
931 'only_matching': True,
932 },
933 {
934 # Video with yt:stretch=17:0
935 'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
936 'info_dict': {
937 'id': 'Q39EVAstoRM',
938 'ext': 'mp4',
939 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
940 'description': 'md5:ee18a25c350637c8faff806845bddee9',
941 'upload_date': '20151107',
942 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
943 'uploader': 'CH GAMER DROID',
944 },
945 'params': {
946 'skip_download': True,
947 },
948 'skip': 'This video does not exist.',
949 },
950 {
951 # Video licensed under Creative Commons
952 'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
953 'info_dict': {
954 'id': 'M4gD1WSo5mA',
955 'ext': 'mp4',
956 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
957 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
958 'duration': 721,
959 'upload_date': '20150127',
960 'uploader_id': 'BerkmanCenter',
961 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
962 'uploader': 'The Berkman Klein Center for Internet & Society',
963 'license': 'Creative Commons Attribution license (reuse allowed)',
964 },
965 'params': {
966 'skip_download': True,
967 },
968 },
969 {
970 # Channel-like uploader_url
971 'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
972 'info_dict': {
973 'id': 'eQcmzGIKrzg',
974 'ext': 'mp4',
975 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
976 'description': 'md5:dda0d780d5a6e120758d1711d062a867',
977 'duration': 4060,
978 'upload_date': '20151119',
979 'uploader': 'Bernie Sanders',
980 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
981 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
982 'license': 'Creative Commons Attribution license (reuse allowed)',
983 },
984 'params': {
985 'skip_download': True,
986 },
987 },
988 {
989 'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
990 'only_matching': True,
991 },
992 {
993 # YouTube Red paid video (https://github.com/rg3/youtube-dl/issues/10059)
994 'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
995 'only_matching': True,
996 },
997 {
998 # Rental video preview
999 'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1000 'info_dict': {
1001 'id': 'uGpuVWrhIzE',
1002 'ext': 'mp4',
1003 'title': 'Piku - Trailer',
1004 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1005 'upload_date': '20150811',
1006 'uploader': 'FlixMatrix',
1007 'uploader_id': 'FlixMatrixKaravan',
1008 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1009 'license': 'Standard YouTube License',
1010 },
1011 'params': {
1012 'skip_download': True,
1013 },
1014 'skip': 'This video is not available.',
1015 },
1016 {
1017 # YouTube Red video with episode data
1018 'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1019 'info_dict': {
1020 'id': 'iqKdEhx-dD4',
1021 'ext': 'mp4',
1022 'title': 'Isolation - Mind Field (Ep 1)',
1023 'description': 'md5:25b78d2f64ae81719f5c96319889b736',
1024 'duration': 2085,
1025 'upload_date': '20170118',
1026 'uploader': 'Vsauce',
1027 'uploader_id': 'Vsauce',
1028 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1029 'license': 'Standard YouTube License',
1030 'series': 'Mind Field',
1031 'season_number': 1,
1032 'episode_number': 1,
1033 },
1034 'params': {
1035 'skip_download': True,
1036 },
1037 'expected_warnings': [
1038 'Skipping DASH manifest',
1039 ],
1040 },
1041 {
1042 # The following content has been identified by the YouTube community
1043 # as inappropriate or offensive to some audiences.
1044 'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1045 'info_dict': {
1046 'id': '6SJNVb0GnPI',
1047 'ext': 'mp4',
1048 'title': 'Race Differences in Intelligence',
1049 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1050 'duration': 965,
1051 'upload_date': '20140124',
1052 'uploader': 'New Century Foundation',
1053 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1054 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1055 'license': 'Standard YouTube License',
1056 },
1057 'params': {
1058 'skip_download': True,
1059 },
1060 },
1061 {
1062 # itag 212
1063 'url': '1t24XAntNCY',
1064 'only_matching': True,
1065 },
1066 {
1067 # geo restricted to JP
1068 'url': 'sJL6WA-aGkQ',
1069 'only_matching': True,
1070 },
1071 {
1072 'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
1073 'only_matching': True,
1074 },
1075 {
1076 'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1077 'only_matching': True,
1078 },
1079 ]
1080
1081 def __init__(self, *args, **kwargs):
1082 super(YoutubeIE, self).__init__(*args, **kwargs)
1083 self._player_cache = {}
1084
1085 def report_video_info_webpage_download(self, video_id):
1086 """Report attempt to download video info webpage."""
1087 self.to_screen('%s: Downloading video info webpage' % video_id)
1088
1089 def report_information_extraction(self, video_id):
1090 """Report attempt to extract video information."""
1091 self.to_screen('%s: Extracting video information' % video_id)
1092
1093 def report_unavailable_format(self, video_id, format):
1094 """Report extracted video URL."""
1095 self.to_screen('%s: Format %s not available' % (video_id, format))
1096
1097 def report_rtmp_download(self):
1098 """Indicate the download will use the RTMP protocol."""
1099 self.to_screen('RTMP download detected')
1100
1101 def _signature_cache_id(self, example_sig):
1102 """ Return a string representation of a signature """
1103 return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
1104
1105 def _extract_signature_function(self, video_id, player_url, example_sig):
1106 id_m = re.match(
1107 r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
1108 player_url)
1109 if not id_m:
1110 raise ExtractorError('Cannot identify player %r' % player_url)
1111 player_type = id_m.group('ext')
1112 player_id = id_m.group('id')
1113
1114 # Read from filesystem cache
1115 func_id = '%s_%s_%s' % (
1116 player_type, player_id, self._signature_cache_id(example_sig))
1117 assert os.path.basename(func_id) == func_id
1118
1119 cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
1120 if cache_spec is not None:
1121 return lambda s: ''.join(s[i] for i in cache_spec)
1122
1123 download_note = (
1124 'Downloading player %s' % player_url
1125 if self._downloader.params.get('verbose') else
1126 'Downloading %s player %s' % (player_type, player_id)
1127 )
1128 if player_type == 'js':
1129 code = self._download_webpage(
1130 player_url, video_id,
1131 note=download_note,
1132 errnote='Download of %s failed' % player_url)
1133 res = self._parse_sig_js(code)
1134 elif player_type == 'swf':
1135 urlh = self._request_webpage(
1136 player_url, video_id,
1137 note=download_note,
1138 errnote='Download of %s failed' % player_url)
1139 code = urlh.read()
1140 res = self._parse_sig_swf(code)
1141 else:
1142 assert False, 'Invalid player type %r' % player_type
1143
1144 test_string = ''.join(map(compat_chr, range(len(example_sig))))
1145 cache_res = res(test_string)
1146 cache_spec = [ord(c) for c in cache_res]
1147
1148 self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
1149 return res
1150
1151 def _print_sig_code(self, func, example_sig):
1152 def gen_sig_code(idxs):
1153 def _genslice(start, end, step):
1154 starts = '' if start == 0 else str(start)
1155 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
1156 steps = '' if step == 1 else (':%d' % step)
1157 return 's[%s%s%s]' % (starts, ends, steps)
1158
1159 step = None
1160 # Quelch pyflakes warnings - start will be set when step is set
1161 start = '(Never used)'
1162 for i, prev in zip(idxs[1:], idxs[:-1]):
1163 if step is not None:
1164 if i - prev == step:
1165 continue
1166 yield _genslice(start, prev, step)
1167 step = None
1168 continue
1169 if i - prev in [-1, 1]:
1170 step = i - prev
1171 start = prev
1172 continue
1173 else:
1174 yield 's[%d]' % prev
1175 if step is None:
1176 yield 's[%d]' % i
1177 else:
1178 yield _genslice(start, i, step)
1179
1180 test_string = ''.join(map(compat_chr, range(len(example_sig))))
1181 cache_res = func(test_string)
1182 cache_spec = [ord(c) for c in cache_res]
1183 expr_code = ' + '.join(gen_sig_code(cache_spec))
1184 signature_id_tuple = '(%s)' % (
1185 ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1186 code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1187 ' return %s\n') % (signature_id_tuple, expr_code)
1188 self.to_screen('Extracted signature function:\n' + code)
1189
1190 def _parse_sig_js(self, jscode):
1191 funcname = self._search_regex(
1192 (r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1193 r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
1194 r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*c\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1195 r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1196 r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
1197 jscode, 'Initial JS player signature function name', group='sig')
1198
1199 jsi = JSInterpreter(jscode)
1200 initial_function = jsi.extract_function(funcname)
1201 return lambda s: initial_function([s])
1202
1203 def _parse_sig_swf(self, file_contents):
1204 swfi = SWFInterpreter(file_contents)
1205 TARGET_CLASSNAME = 'SignatureDecipher'
1206 searched_class = swfi.extract_class(TARGET_CLASSNAME)
1207 initial_function = swfi.extract_function(searched_class, 'decipher')
1208 return lambda s: initial_function([s])
1209
1210 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1211 """Turn the encrypted s field into a working signature"""
1212
1213 if player_url is None:
1214 raise ExtractorError('Cannot decrypt signature without player_url')
1215
1216 if player_url.startswith('//'):
1217 player_url = 'https:' + player_url
1218 elif not re.match(r'https?://', player_url):
1219 player_url = compat_urlparse.urljoin(
1220 'https://www.youtube.com', player_url)
1221 try:
1222 player_id = (player_url, self._signature_cache_id(s))
1223 if player_id not in self._player_cache:
1224 func = self._extract_signature_function(
1225 video_id, player_url, s
1226 )
1227 self._player_cache[player_id] = func
1228 func = self._player_cache[player_id]
1229 if self._downloader.params.get('youtube_print_sig_code'):
1230 self._print_sig_code(func, s)
1231 return func(s)
1232 except Exception as e:
1233 tb = traceback.format_exc()
1234 raise ExtractorError(
1235 'Signature extraction failed: ' + tb, cause=e)
1236
1237 def _get_subtitles(self, video_id, webpage):
1238 try:
1239 subs_doc = self._download_xml(
1240 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1241 video_id, note=False)
1242 except ExtractorError as err:
1243 self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
1244 return {}
1245
1246 sub_lang_list = {}
1247 for track in subs_doc.findall('track'):
1248 lang = track.attrib['lang_code']
1249 if lang in sub_lang_list:
1250 continue
1251 sub_formats = []
1252 for ext in self._SUBTITLE_FORMATS:
1253 params = compat_urllib_parse_urlencode({
1254 'lang': lang,
1255 'v': video_id,
1256 'fmt': ext,
1257 'name': track.attrib['name'].encode('utf-8'),
1258 })
1259 sub_formats.append({
1260 'url': 'https://www.youtube.com/api/timedtext?' + params,
1261 'ext': ext,
1262 })
1263 sub_lang_list[lang] = sub_formats
1264 if not sub_lang_list:
1265 self._downloader.report_warning('video doesn\'t have subtitles')
1266 return {}
1267 return sub_lang_list
1268
1269 def _get_ytplayer_config(self, video_id, webpage):
1270 patterns = (
1271 # User data may contain arbitrary character sequences that may affect
1272 # JSON extraction with regex, e.g. when '};' is contained the second
1273 # regex won't capture the whole JSON. Yet working around by trying more
1274 # concrete regex first keeping in mind proper quoted string handling
1275 # to be implemented in future that will replace this workaround (see
1276 # https://github.com/rg3/youtube-dl/issues/7468,
1277 # https://github.com/rg3/youtube-dl/pull/7599)
1278 r';ytplayer\.config\s*=\s*({.+?});ytplayer',
1279 r';ytplayer\.config\s*=\s*({.+?});',
1280 )
1281 config = self._search_regex(
1282 patterns, webpage, 'ytplayer.config', default=None)
1283 if config:
1284 return self._parse_json(
1285 uppercase_escape(config), video_id, fatal=False)
1286
1287 def _get_automatic_captions(self, video_id, webpage):
1288 """We need the webpage for getting the captions url, pass it as an
1289 argument to speed up the process."""
1290 self.to_screen('%s: Looking for automatic captions' % video_id)
1291 player_config = self._get_ytplayer_config(video_id, webpage)
1292 err_msg = 'Couldn\'t find automatic captions for %s' % video_id
1293 if not player_config:
1294 self._downloader.report_warning(err_msg)
1295 return {}
1296 try:
1297 args = player_config['args']
1298 caption_url = args.get('ttsurl')
1299 if caption_url:
1300 timestamp = args['timestamp']
1301 # We get the available subtitles
1302 list_params = compat_urllib_parse_urlencode({
1303 'type': 'list',
1304 'tlangs': 1,
1305 'asrs': 1,
1306 })
1307 list_url = caption_url + '&' + list_params
1308 caption_list = self._download_xml(list_url, video_id)
1309 original_lang_node = caption_list.find('track')
1310 if original_lang_node is None:
1311 self._downloader.report_warning('Video doesn\'t have automatic captions')
1312 return {}
1313 original_lang = original_lang_node.attrib['lang_code']
1314 caption_kind = original_lang_node.attrib.get('kind', '')
1315
1316 sub_lang_list = {}
1317 for lang_node in caption_list.findall('target'):
1318 sub_lang = lang_node.attrib['lang_code']
1319 sub_formats = []
1320 for ext in self._SUBTITLE_FORMATS:
1321 params = compat_urllib_parse_urlencode({
1322 'lang': original_lang,
1323 'tlang': sub_lang,
1324 'fmt': ext,
1325 'ts': timestamp,
1326 'kind': caption_kind,
1327 })
1328 sub_formats.append({
1329 'url': caption_url + '&' + params,
1330 'ext': ext,
1331 })
1332 sub_lang_list[sub_lang] = sub_formats
1333 return sub_lang_list
1334
1335 def make_captions(sub_url, sub_langs):
1336 parsed_sub_url = compat_urllib_parse_urlparse(sub_url)
1337 caption_qs = compat_parse_qs(parsed_sub_url.query)
1338 captions = {}
1339 for sub_lang in sub_langs:
1340 sub_formats = []
1341 for ext in self._SUBTITLE_FORMATS:
1342 caption_qs.update({
1343 'tlang': [sub_lang],
1344 'fmt': [ext],
1345 })
1346 sub_url = compat_urlparse.urlunparse(parsed_sub_url._replace(
1347 query=compat_urllib_parse_urlencode(caption_qs, True)))
1348 sub_formats.append({
1349 'url': sub_url,
1350 'ext': ext,
1351 })
1352 captions[sub_lang] = sub_formats
1353 return captions
1354
1355 # New captions format as of 22.06.2017
1356 player_response = args.get('player_response')
1357 if player_response and isinstance(player_response, compat_str):
1358 player_response = self._parse_json(
1359 player_response, video_id, fatal=False)
1360 if player_response:
1361 renderer = player_response['captions']['playerCaptionsTracklistRenderer']
1362 base_url = renderer['captionTracks'][0]['baseUrl']
1363 sub_lang_list = []
1364 for lang in renderer['translationLanguages']:
1365 lang_code = lang.get('languageCode')
1366 if lang_code:
1367 sub_lang_list.append(lang_code)
1368 return make_captions(base_url, sub_lang_list)
1369
1370 # Some videos don't provide ttsurl but rather caption_tracks and
1371 # caption_translation_languages (e.g. 20LmZk1hakA)
1372 # Does not used anymore as of 22.06.2017
1373 caption_tracks = args['caption_tracks']
1374 caption_translation_languages = args['caption_translation_languages']
1375 caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
1376 sub_lang_list = []
1377 for lang in caption_translation_languages.split(','):
1378 lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
1379 sub_lang = lang_qs.get('lc', [None])[0]
1380 if sub_lang:
1381 sub_lang_list.append(sub_lang)
1382 return make_captions(caption_url, sub_lang_list)
1383 # An extractor error can be raise by the download process if there are
1384 # no automatic captions but there are subtitles
1385 except (KeyError, IndexError, ExtractorError):
1386 self._downloader.report_warning(err_msg)
1387 return {}
1388
1389 def _mark_watched(self, video_id, video_info):
1390 playback_url = video_info.get('videostats_playback_base_url', [None])[0]
1391 if not playback_url:
1392 return
1393 parsed_playback_url = compat_urlparse.urlparse(playback_url)
1394 qs = compat_urlparse.parse_qs(parsed_playback_url.query)
1395
1396 # cpn generation algorithm is reverse engineered from base.js.
1397 # In fact it works even with dummy cpn.
1398 CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1399 cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
1400
1401 qs.update({
1402 'ver': ['2'],
1403 'cpn': [cpn],
1404 })
1405 playback_url = compat_urlparse.urlunparse(
1406 parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
1407
1408 self._download_webpage(
1409 playback_url, video_id, 'Marking watched',
1410 'Unable to mark watched', fatal=False)
1411
1412 @staticmethod
1413 def _extract_urls(webpage):
1414 # Embedded YouTube player
1415 entries = [
1416 unescapeHTML(mobj.group('url'))
1417 for mobj in re.finditer(r'''(?x)
1418 (?:
1419 <iframe[^>]+?src=|
1420 data-video-url=|
1421 <embed[^>]+?src=|
1422 embedSWF\(?:\s*|
1423 <object[^>]+data=|
1424 new\s+SWFObject\(
1425 )
1426 (["\'])
1427 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1428 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1429 \1''', webpage)]
1430
1431 # lazyYT YouTube embed
1432 entries.extend(list(map(
1433 unescapeHTML,
1434 re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
1435
1436 # Wordpress "YouTube Video Importer" plugin
1437 matches = re.findall(r'''(?x)<div[^>]+
1438 class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1439 data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1440 entries.extend(m[-1] for m in matches)
1441
1442 return entries
1443
1444 @staticmethod
1445 def _extract_url(webpage):
1446 urls = YoutubeIE._extract_urls(webpage)
1447 return urls[0] if urls else None
1448
1449 @classmethod
1450 def extract_id(cls, url):
1451 mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1452 if mobj is None:
1453 raise ExtractorError('Invalid URL: %s' % url)
1454 video_id = mobj.group(2)
1455 return video_id
1456
1457 def _extract_annotations(self, video_id):
1458 url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1459 return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
1460
1461 @staticmethod
1462 def _extract_chapters(description, duration):
1463 if not description:
1464 return None
1465 chapter_lines = re.findall(
1466 r'(?:^|<br\s*/>)([^<]*<a[^>]+onclick=["\']yt\.www\.watch\.player\.seekTo[^>]+>(\d{1,2}:\d{1,2}(?::\d{1,2})?)</a>[^>]*)(?=$|<br\s*/>)',
1467 description)
1468 if not chapter_lines:
1469 return None
1470 chapters = []
1471 for next_num, (chapter_line, time_point) in enumerate(
1472 chapter_lines, start=1):
1473 start_time = parse_duration(time_point)
1474 if start_time is None:
1475 continue
1476 if start_time > duration:
1477 break
1478 end_time = (duration if next_num == len(chapter_lines)
1479 else parse_duration(chapter_lines[next_num][1]))
1480 if end_time is None:
1481 continue
1482 if end_time > duration:
1483 end_time = duration
1484 if start_time > end_time:
1485 break
1486 chapter_title = re.sub(
1487 r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
1488 chapter_title = re.sub(r'\s+', ' ', chapter_title)
1489 chapters.append({
1490 'start_time': start_time,
1491 'end_time': end_time,
1492 'title': chapter_title,
1493 })
1494 return chapters
1495
1496 def _real_extract(self, url):
1497 url, smuggled_data = unsmuggle_url(url, {})
1498
1499 proto = (
1500 'http' if self._downloader.params.get('prefer_insecure', False)
1501 else 'https')
1502
1503 start_time = None
1504 end_time = None
1505 parsed_url = compat_urllib_parse_urlparse(url)
1506 for component in [parsed_url.fragment, parsed_url.query]:
1507 query = compat_parse_qs(component)
1508 if start_time is None and 't' in query:
1509 start_time = parse_duration(query['t'][0])
1510 if start_time is None and 'start' in query:
1511 start_time = parse_duration(query['start'][0])
1512 if end_time is None and 'end' in query:
1513 end_time = parse_duration(query['end'][0])
1514
1515 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1516 mobj = re.search(self._NEXT_URL_RE, url)
1517 if mobj:
1518 url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
1519 video_id = self.extract_id(url)
1520
1521 # Get video webpage
1522 url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1523 video_webpage = self._download_webpage(url, video_id)
1524
1525 # Attempt to extract SWF player URL
1526 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1527 if mobj is not None:
1528 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1529 else:
1530 player_url = None
1531
1532 dash_mpds = []
1533
1534 def add_dash_mpd(video_info):
1535 dash_mpd = video_info.get('dashmpd')
1536 if dash_mpd and dash_mpd[0] not in dash_mpds:
1537 dash_mpds.append(dash_mpd[0])
1538
1539 is_live = None
1540 view_count = None
1541
1542 def extract_view_count(v_info):
1543 return int_or_none(try_get(v_info, lambda x: x['view_count'][0]))
1544
1545 player_response = {}
1546
1547 # Get video info
1548 embed_webpage = None
1549 if re.search(r'player-age-gate-content">', video_webpage) is not None:
1550 age_gate = True
1551 # We simulate the access to the video from www.youtube.com/v/{video_id}
1552 # this can be viewed without login into Youtube
1553 url = proto + '://www.youtube.com/embed/%s' % video_id
1554 embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
1555 data = compat_urllib_parse_urlencode({
1556 'video_id': video_id,
1557 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1558 'sts': self._search_regex(
1559 r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
1560 })
1561 video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1562 video_info_webpage = self._download_webpage(
1563 video_info_url, video_id,
1564 note='Refetching age-gated info webpage',
1565 errnote='unable to download video info webpage')
1566 video_info = compat_parse_qs(video_info_webpage)
1567 add_dash_mpd(video_info)
1568 else:
1569 age_gate = False
1570 video_info = None
1571 sts = None
1572 # Try looking directly into the video webpage
1573 ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
1574 if ytplayer_config:
1575 args = ytplayer_config['args']
1576 if args.get('url_encoded_fmt_stream_map') or args.get('hlsvp'):
1577 # Convert to the same format returned by compat_parse_qs
1578 video_info = dict((k, [v]) for k, v in args.items())
1579 add_dash_mpd(video_info)
1580 # Rental video is not rented but preview is available (e.g.
1581 # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
1582 # https://github.com/rg3/youtube-dl/issues/10532)
1583 if not video_info and args.get('ypc_vid'):
1584 return self.url_result(
1585 args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
1586 if args.get('livestream') == '1' or args.get('live_playback') == 1:
1587 is_live = True
1588 sts = ytplayer_config.get('sts')
1589 if not player_response:
1590 pl_response = str_or_none(args.get('player_response'))
1591 if pl_response:
1592 pl_response = self._parse_json(pl_response, video_id, fatal=False)
1593 if isinstance(pl_response, dict):
1594 player_response = pl_response
1595 if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
1596 # We also try looking in get_video_info since it may contain different dashmpd
1597 # URL that points to a DASH manifest with possibly different itag set (some itags
1598 # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
1599 # manifest pointed by get_video_info's dashmpd).
1600 # The general idea is to take a union of itags of both DASH manifests (for example
1601 # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
1602 self.report_video_info_webpage_download(video_id)
1603 for el in ('info', 'embedded', 'detailpage', 'vevo', ''):
1604 query = {
1605 'video_id': video_id,
1606 'ps': 'default',
1607 'eurl': '',
1608 'gl': 'US',
1609 'hl': 'en',
1610 }
1611 if el:
1612 query['el'] = el
1613 if sts:
1614 query['sts'] = sts
1615 video_info_webpage = self._download_webpage(
1616 '%s://www.youtube.com/get_video_info' % proto,
1617 video_id, note=False,
1618 errnote='unable to download video info webpage',
1619 fatal=False, query=query)
1620 if not video_info_webpage:
1621 continue
1622 get_video_info = compat_parse_qs(video_info_webpage)
1623 if not player_response:
1624 pl_response = get_video_info.get('player_response', [None])[0]
1625 if isinstance(pl_response, dict):
1626 player_response = pl_response
1627 add_dash_mpd(get_video_info)
1628 if view_count is None:
1629 view_count = extract_view_count(get_video_info)
1630 if not video_info:
1631 video_info = get_video_info
1632 if 'token' in get_video_info:
1633 # Different get_video_info requests may report different results, e.g.
1634 # some may report video unavailability, but some may serve it without
1635 # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
1636 # the original webpage as well as el=info and el=embedded get_video_info
1637 # requests report video unavailability due to geo restriction while
1638 # el=detailpage succeeds and returns valid data). This is probably
1639 # due to YouTube measures against IP ranges of hosting providers.
1640 # Working around by preferring the first succeeded video_info containing
1641 # the token if no such video_info yet was found.
1642 if 'token' not in video_info:
1643 video_info = get_video_info
1644 break
1645
1646 def extract_unavailable_message():
1647 return self._html_search_regex(
1648 r'(?s)<h1[^>]+id="unavailable-message"[^>]*>(.+?)</h1>',
1649 video_webpage, 'unavailable message', default=None)
1650
1651 if 'token' not in video_info:
1652 if 'reason' in video_info:
1653 if 'The uploader has not made this video available in your country.' in video_info['reason']:
1654 regions_allowed = self._html_search_meta(
1655 'regionsAllowed', video_webpage, default=None)
1656 countries = regions_allowed.split(',') if regions_allowed else None
1657 self.raise_geo_restricted(
1658 msg=video_info['reason'][0], countries=countries)
1659 reason = video_info['reason'][0]
1660 if 'Invalid parameters' in reason:
1661 unavailable_message = extract_unavailable_message()
1662 if unavailable_message:
1663 reason = unavailable_message
1664 raise ExtractorError(
1665 'YouTube said: %s' % reason,
1666 expected=True, video_id=video_id)
1667 else:
1668 raise ExtractorError(
1669 '"token" parameter not in video info for unknown reason',
1670 video_id=video_id)
1671
1672 video_details = try_get(
1673 player_response, lambda x: x['videoDetails'], dict) or {}
1674
1675 # title
1676 if 'title' in video_info:
1677 video_title = video_info['title'][0]
1678 elif 'title' in player_response:
1679 video_title = video_details['title']
1680 else:
1681 self._downloader.report_warning('Unable to extract video title')
1682 video_title = '_'
1683
1684 # description
1685 description_original = video_description = get_element_by_id("eow-description", video_webpage)
1686 if video_description:
1687
1688 def replace_url(m):
1689 redir_url = compat_urlparse.urljoin(url, m.group(1))
1690 parsed_redir_url = compat_urllib_parse_urlparse(redir_url)
1691 if re.search(r'^(?:www\.)?(?:youtube(?:-nocookie)?\.com|youtu\.be)$', parsed_redir_url.netloc) and parsed_redir_url.path == '/redirect':
1692 qs = compat_parse_qs(parsed_redir_url.query)
1693 q = qs.get('q')
1694 if q and q[0]:
1695 return q[0]
1696 return redir_url
1697
1698 description_original = video_description = re.sub(r'''(?x)
1699 <a\s+
1700 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1701 (?:title|href)="([^"]+)"\s+
1702 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1703 class="[^"]*"[^>]*>
1704 [^<]+\.{3}\s*
1705 </a>
1706 ''', replace_url, video_description)
1707 video_description = clean_html(video_description)
1708 else:
1709 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1710 if fd_mobj:
1711 video_description = unescapeHTML(fd_mobj.group(1))
1712 else:
1713 video_description = ''
1714
1715 if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
1716 if not self._downloader.params.get('noplaylist'):
1717 entries = []
1718 feed_ids = []
1719 multifeed_metadata_list = video_info['multifeed_metadata_list'][0]
1720 for feed in multifeed_metadata_list.split(','):
1721 # Unquote should take place before split on comma (,) since textual
1722 # fields may contain comma as well (see
1723 # https://github.com/rg3/youtube-dl/issues/8536)
1724 feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
1725 entries.append({
1726 '_type': 'url_transparent',
1727 'ie_key': 'Youtube',
1728 'url': smuggle_url(
1729 '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
1730 {'force_singlefeed': True}),
1731 'title': '%s (%s)' % (video_title, feed_data['title'][0]),
1732 })
1733 feed_ids.append(feed_data['id'][0])
1734 self.to_screen(
1735 'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1736 % (', '.join(feed_ids), video_id))
1737 return self.playlist_result(entries, video_id, video_title, video_description)
1738 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1739
1740 if view_count is None:
1741 view_count = extract_view_count(video_info)
1742 if view_count is None and video_details:
1743 view_count = int_or_none(video_details.get('viewCount'))
1744
1745 # Check for "rental" videos
1746 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1747 raise ExtractorError('"rental" videos not supported. See https://github.com/rg3/youtube-dl/issues/359 for more information.', expected=True)
1748
1749 def _extract_filesize(media_url):
1750 return int_or_none(self._search_regex(
1751 r'\bclen[=/](\d+)', media_url, 'filesize', default=None))
1752
1753 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1754 self.report_rtmp_download()
1755 formats = [{
1756 'format_id': '_rtmp',
1757 'protocol': 'rtmp',
1758 'url': video_info['conn'][0],
1759 'player_url': player_url,
1760 }]
1761 elif not is_live and (len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1):
1762 encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
1763 if 'rtmpe%3Dyes' in encoded_url_map:
1764 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1765 formats_spec = {}
1766 fmt_list = video_info.get('fmt_list', [''])[0]
1767 if fmt_list:
1768 for fmt in fmt_list.split(','):
1769 spec = fmt.split('/')
1770 if len(spec) > 1:
1771 width_height = spec[1].split('x')
1772 if len(width_height) == 2:
1773 formats_spec[spec[0]] = {
1774 'resolution': spec[1],
1775 'width': int_or_none(width_height[0]),
1776 'height': int_or_none(width_height[1]),
1777 }
1778 q = qualities(['small', 'medium', 'hd720'])
1779 formats = []
1780 for url_data_str in encoded_url_map.split(','):
1781 url_data = compat_parse_qs(url_data_str)
1782 if 'itag' not in url_data or 'url' not in url_data:
1783 continue
1784 format_id = url_data['itag'][0]
1785 url = url_data['url'][0]
1786
1787 if 's' in url_data or self._downloader.params.get('youtube_include_dash_manifest', True):
1788 ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
1789 jsplayer_url_json = self._search_regex(
1790 ASSETS_RE,
1791 embed_webpage if age_gate else video_webpage,
1792 'JS player URL (1)', default=None)
1793 if not jsplayer_url_json and not age_gate:
1794 # We need the embed website after all
1795 if embed_webpage is None:
1796 embed_url = proto + '://www.youtube.com/embed/%s' % video_id
1797 embed_webpage = self._download_webpage(
1798 embed_url, video_id, 'Downloading embed webpage')
1799 jsplayer_url_json = self._search_regex(
1800 ASSETS_RE, embed_webpage, 'JS player URL')
1801
1802 player_url = json.loads(jsplayer_url_json)
1803 if player_url is None:
1804 player_url_json = self._search_regex(
1805 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
1806 video_webpage, 'age gate player URL')
1807 player_url = json.loads(player_url_json)
1808
1809 if 'sig' in url_data:
1810 url += '&signature=' + url_data['sig'][0]
1811 elif 's' in url_data:
1812 encrypted_sig = url_data['s'][0]
1813
1814 if self._downloader.params.get('verbose'):
1815 if player_url is None:
1816 player_version = 'unknown'
1817 player_desc = 'unknown'
1818 else:
1819 if player_url.endswith('swf'):
1820 player_version = self._search_regex(
1821 r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
1822 'flash player', fatal=False)
1823 player_desc = 'flash player %s' % player_version
1824 else:
1825 player_version = self._search_regex(
1826 [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
1827 r'(?:www|player)-([^/]+)(?:/[a-z]{2}_[A-Z]{2})?/base\.js'],
1828 player_url,
1829 'html5 player', fatal=False)
1830 player_desc = 'html5 player %s' % player_version
1831
1832 parts_sizes = self._signature_cache_id(encrypted_sig)
1833 self.to_screen('{%s} signature length %s, %s' %
1834 (format_id, parts_sizes, player_desc))
1835
1836 signature = self._decrypt_signature(
1837 encrypted_sig, video_id, player_url, age_gate)
1838 url += '&signature=' + signature
1839 if 'ratebypass' not in url:
1840 url += '&ratebypass=yes'
1841
1842 dct = {
1843 'format_id': format_id,
1844 'url': url,
1845 'player_url': player_url,
1846 }
1847 if format_id in self._formats:
1848 dct.update(self._formats[format_id])
1849 if format_id in formats_spec:
1850 dct.update(formats_spec[format_id])
1851
1852 # Some itags are not included in DASH manifest thus corresponding formats will
1853 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
1854 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
1855 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
1856 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
1857
1858 filesize = int_or_none(url_data.get(
1859 'clen', [None])[0]) or _extract_filesize(url)
1860
1861 quality = url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0]
1862
1863 more_fields = {
1864 'filesize': filesize,
1865 'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
1866 'width': width,
1867 'height': height,
1868 'fps': int_or_none(url_data.get('fps', [None])[0]),
1869 'format_note': quality,
1870 'quality': q(quality),
1871 }
1872 for key, value in more_fields.items():
1873 if value:
1874 dct[key] = value
1875 type_ = url_data.get('type', [None])[0]
1876 if type_:
1877 type_split = type_.split(';')
1878 kind_ext = type_split[0].split('/')
1879 if len(kind_ext) == 2:
1880 kind, _ = kind_ext
1881 dct['ext'] = mimetype2ext(type_split[0])
1882 if kind in ('audio', 'video'):
1883 codecs = None
1884 for mobj in re.finditer(
1885 r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
1886 if mobj.group('key') == 'codecs':
1887 codecs = mobj.group('val')
1888 break
1889 if codecs:
1890 dct.update(parse_codecs(codecs))
1891 if dct.get('acodec') == 'none' or dct.get('vcodec') == 'none':
1892 dct['downloader_options'] = {
1893 # Youtube throttles chunks >~10M
1894 'http_chunk_size': 10485760,
1895 }
1896 formats.append(dct)
1897 elif video_info.get('hlsvp'):
1898 manifest_url = video_info['hlsvp'][0]
1899 formats = []
1900 m3u8_formats = self._extract_m3u8_formats(
1901 manifest_url, video_id, 'mp4', fatal=False)
1902 for a_format in m3u8_formats:
1903 itag = self._search_regex(
1904 r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
1905 if itag:
1906 a_format['format_id'] = itag
1907 if itag in self._formats:
1908 dct = self._formats[itag].copy()
1909 dct.update(a_format)
1910 a_format = dct
1911 a_format['player_url'] = player_url
1912 # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
1913 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
1914 formats.append(a_format)
1915 else:
1916 error_message = clean_html(video_info.get('reason', [None])[0])
1917 if not error_message:
1918 error_message = extract_unavailable_message()
1919 if error_message:
1920 raise ExtractorError(error_message, expected=True)
1921 raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1922
1923 # uploader
1924 video_uploader = try_get(
1925 video_info, lambda x: x['author'][0],
1926 compat_str) or str_or_none(video_details.get('author'))
1927 if video_uploader:
1928 video_uploader = compat_urllib_parse_unquote_plus(video_uploader)
1929 else:
1930 self._downloader.report_warning('unable to extract uploader name')
1931
1932 # uploader_id
1933 video_uploader_id = None
1934 video_uploader_url = None
1935 mobj = re.search(
1936 r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
1937 video_webpage)
1938 if mobj is not None:
1939 video_uploader_id = mobj.group('uploader_id')
1940 video_uploader_url = mobj.group('uploader_url')
1941 else:
1942 self._downloader.report_warning('unable to extract uploader nickname')
1943
1944 channel_id = self._html_search_meta(
1945 'channelId', video_webpage, 'channel id')
1946 channel_url = 'http://www.youtube.com/channel/%s' % channel_id if channel_id else None
1947
1948 # thumbnail image
1949 # We try first to get a high quality image:
1950 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1951 video_webpage, re.DOTALL)
1952 if m_thumb is not None:
1953 video_thumbnail = m_thumb.group(1)
1954 elif 'thumbnail_url' not in video_info:
1955 self._downloader.report_warning('unable to extract video thumbnail')
1956 video_thumbnail = None
1957 else: # don't panic if we can't find it
1958 video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
1959
1960 # upload date
1961 upload_date = self._html_search_meta(
1962 'datePublished', video_webpage, 'upload date', default=None)
1963 if not upload_date:
1964 upload_date = self._search_regex(
1965 [r'(?s)id="eow-date.*?>(.*?)</span>',
1966 r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']'],
1967 video_webpage, 'upload date', default=None)
1968 upload_date = unified_strdate(upload_date)
1969
1970 video_license = self._html_search_regex(
1971 r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
1972 video_webpage, 'license', default=None)
1973
1974 m_music = re.search(
1975 r'''(?x)
1976 <h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*
1977 <ul[^>]*>\s*
1978 <li>(?P<title>.+?)
1979 by (?P<creator>.+?)
1980 (?:
1981 \(.+?\)|
1982 <a[^>]*
1983 (?:
1984 \bhref=["\']/red[^>]*>| # drop possible
1985 >\s*Listen ad-free with YouTube Red # YouTube Red ad
1986 )
1987 .*?
1988 )?</li
1989 ''',
1990 video_webpage)
1991 if m_music:
1992 video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
1993 video_creator = clean_html(m_music.group('creator'))
1994 else:
1995 video_alt_title = video_creator = None
1996
1997 def extract_meta(field):
1998 return self._html_search_regex(
1999 r'<h4[^>]+class="title"[^>]*>\s*%s\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li>\s*' % field,
2000 video_webpage, field, default=None)
2001
2002 track = extract_meta('Song')
2003 artist = extract_meta('Artist')
2004
2005 m_episode = re.search(
2006 r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*ā€¢\s*E(?P<episode>\d+)</span>',
2007 video_webpage)
2008 if m_episode:
2009 series = m_episode.group('series')
2010 season_number = int(m_episode.group('season'))
2011 episode_number = int(m_episode.group('episode'))
2012 else:
2013 series = season_number = episode_number = None
2014
2015 m_cat_container = self._search_regex(
2016 r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
2017 video_webpage, 'categories', default=None)
2018 if m_cat_container:
2019 category = self._html_search_regex(
2020 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
2021 default=None)
2022 video_categories = None if category is None else [category]
2023 else:
2024 video_categories = None
2025
2026 video_tags = [
2027 unescapeHTML(m.group('content'))
2028 for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
2029
2030 def _extract_count(count_name):
2031 return str_to_int(self._search_regex(
2032 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
2033 % re.escape(count_name),
2034 video_webpage, count_name, default=None))
2035
2036 like_count = _extract_count('like')
2037 dislike_count = _extract_count('dislike')
2038
2039 if view_count is None:
2040 view_count = str_to_int(self._search_regex(
2041 r'<[^>]+class=["\']watch-view-count[^>]+>\s*([\d,\s]+)', video_webpage,
2042 'view count', default=None))
2043
2044 # subtitles
2045 video_subtitles = self.extract_subtitles(video_id, video_webpage)
2046 automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
2047
2048 video_duration = try_get(
2049 video_info, lambda x: int_or_none(x['length_seconds'][0]))
2050 if not video_duration:
2051 video_duration = int_or_none(video_details.get('lengthSeconds'))
2052 if not video_duration:
2053 video_duration = parse_duration(self._html_search_meta(
2054 'duration', video_webpage, 'video duration'))
2055
2056 # annotations
2057 video_annotations = None
2058 if self._downloader.params.get('writeannotations', False):
2059 video_annotations = self._extract_annotations(video_id)
2060
2061 chapters = self._extract_chapters(description_original, video_duration)
2062
2063 # Look for the DASH manifest
2064 if self._downloader.params.get('youtube_include_dash_manifest', True):
2065 dash_mpd_fatal = True
2066 for mpd_url in dash_mpds:
2067 dash_formats = {}
2068 try:
2069 def decrypt_sig(mobj):
2070 s = mobj.group(1)
2071 dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
2072 return '/signature/%s' % dec_s
2073
2074 mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
2075
2076 for df in self._extract_mpd_formats(
2077 mpd_url, video_id, fatal=dash_mpd_fatal,
2078 formats_dict=self._formats):
2079 if not df.get('filesize'):
2080 df['filesize'] = _extract_filesize(df['url'])
2081 # Do not overwrite DASH format found in some previous DASH manifest
2082 if df['format_id'] not in dash_formats:
2083 dash_formats[df['format_id']] = df
2084 # Additional DASH manifests may end up in HTTP Error 403 therefore
2085 # allow them to fail without bug report message if we already have
2086 # some DASH manifest succeeded. This is temporary workaround to reduce
2087 # burst of bug reports until we figure out the reason and whether it
2088 # can be fixed at all.
2089 dash_mpd_fatal = False
2090 except (ExtractorError, KeyError) as e:
2091 self.report_warning(
2092 'Skipping DASH manifest: %r' % e, video_id)
2093 if dash_formats:
2094 # Remove the formats we found through non-DASH, they
2095 # contain less info and it can be wrong, because we use
2096 # fixed values (for example the resolution). See
2097 # https://github.com/rg3/youtube-dl/issues/5774 for an
2098 # example.
2099 formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
2100 formats.extend(dash_formats.values())
2101
2102 # Check for malformed aspect ratio
2103 stretched_m = re.search(
2104 r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
2105 video_webpage)
2106 if stretched_m:
2107 w = float(stretched_m.group('w'))
2108 h = float(stretched_m.group('h'))
2109 # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
2110 # We will only process correct ratios.
2111 if w > 0 and h > 0:
2112 ratio = w / h
2113 for f in formats:
2114 if f.get('vcodec') != 'none':
2115 f['stretched_ratio'] = ratio
2116
2117 self._sort_formats(formats)
2118
2119 self.mark_watched(video_id, video_info)
2120
2121 return {
2122 'id': video_id,
2123 'uploader': video_uploader,
2124 'uploader_id': video_uploader_id,
2125 'uploader_url': video_uploader_url,
2126 'channel_id': channel_id,
2127 'channel_url': channel_url,
2128 'upload_date': upload_date,
2129 'license': video_license,
2130 'creator': video_creator or artist,
2131 'title': video_title,
2132 'alt_title': video_alt_title or track,
2133 'thumbnail': video_thumbnail,
2134 'description': video_description,
2135 'categories': video_categories,
2136 'tags': video_tags,
2137 'subtitles': video_subtitles,
2138 'automatic_captions': automatic_captions,
2139 'duration': video_duration,
2140 'age_limit': 18 if age_gate else 0,
2141 'annotations': video_annotations,
2142 'chapters': chapters,
2143 'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
2144 'view_count': view_count,
2145 'like_count': like_count,
2146 'dislike_count': dislike_count,
2147 'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
2148 'formats': formats,
2149 'is_live': is_live,
2150 'start_time': start_time,
2151 'end_time': end_time,
2152 'series': series,
2153 'season_number': season_number,
2154 'episode_number': episode_number,
2155 'track': track,
2156 'artist': artist,
2157 }
2158
2159
2160 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
2161 IE_DESC = 'YouTube.com playlists'
2162 _VALID_URL = r"""(?x)(?:
2163 (?:https?://)?
2164 (?:\w+\.)?
2165 (?:
2166 (?:
2167 youtube\.com|
2168 invidio\.us
2169 )
2170 /
2171 (?:
2172 (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/(?:videoseries|[0-9A-Za-z_-]{11}))
2173 \? (?:.*?[&;])*? (?:p|a|list)=
2174 | p/
2175 )|
2176 youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
2177 )
2178 (
2179 (?:PL|LL|EC|UU|FL|RD|UL|TL|OLAK5uy_)?[0-9A-Za-z-_]{10,}
2180 # Top tracks, they can also include dots
2181 |(?:MC)[\w\.]*
2182 )
2183 .*
2184 |
2185 (%(playlist_id)s)
2186 )""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
2187 _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
2188 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
2189 IE_NAME = 'youtube:playlist'
2190 _TESTS = [{
2191 'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
2192 'info_dict': {
2193 'title': 'ytdl test PL',
2194 'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
2195 },
2196 'playlist_count': 3,
2197 }, {
2198 'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
2199 'info_dict': {
2200 'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
2201 'title': 'YDL_Empty_List',
2202 },
2203 'playlist_count': 0,
2204 'skip': 'This playlist is private',
2205 }, {
2206 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
2207 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2208 'info_dict': {
2209 'title': '29C3: Not my department',
2210 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2211 },
2212 'playlist_count': 95,
2213 }, {
2214 'note': 'issue #673',
2215 'url': 'PLBB231211A4F62143',
2216 'info_dict': {
2217 'title': '[OLD]Team Fortress 2 (Class-based LP)',
2218 'id': 'PLBB231211A4F62143',
2219 },
2220 'playlist_mincount': 26,
2221 }, {
2222 'note': 'Large playlist',
2223 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
2224 'info_dict': {
2225 'title': 'Uploads from Cauchemar',
2226 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
2227 },
2228 'playlist_mincount': 799,
2229 }, {
2230 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2231 'info_dict': {
2232 'title': 'YDL_safe_search',
2233 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2234 },
2235 'playlist_count': 2,
2236 'skip': 'This playlist is private',
2237 }, {
2238 'note': 'embedded',
2239 'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2240 'playlist_count': 4,
2241 'info_dict': {
2242 'title': 'JODA15',
2243 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2244 }
2245 }, {
2246 'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2247 'playlist_mincount': 485,
2248 'info_dict': {
2249 'title': '2017 čÆčŖžęœ€ę–°å–®ę›² (2/24ꛓꖰ)',
2250 'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2251 }
2252 }, {
2253 'note': 'Embedded SWF player',
2254 'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
2255 'playlist_count': 4,
2256 'info_dict': {
2257 'title': 'JODA7',
2258 'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
2259 }
2260 }, {
2261 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
2262 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
2263 'info_dict': {
2264 'title': 'Uploads from Interstellar Movie',
2265 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
2266 },
2267 'playlist_mincount': 21,
2268 }, {
2269 # Playlist URL that does not actually serve a playlist
2270 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
2271 'info_dict': {
2272 'id': 'FqZTN594JQw',
2273 'ext': 'webm',
2274 'title': "Smiley's People 01 detective, Adventure Series, Action",
2275 'uploader': 'STREEM',
2276 'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
2277 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
2278 'upload_date': '20150526',
2279 'license': 'Standard YouTube License',
2280 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
2281 'categories': ['People & Blogs'],
2282 'tags': list,
2283 'view_count': int,
2284 'like_count': int,
2285 'dislike_count': int,
2286 },
2287 'params': {
2288 'skip_download': True,
2289 },
2290 'add_ie': [YoutubeIE.ie_key()],
2291 }, {
2292 'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
2293 'info_dict': {
2294 'id': 'yeWKywCrFtk',
2295 'ext': 'mp4',
2296 'title': 'Small Scale Baler and Braiding Rugs',
2297 'uploader': 'Backus-Page House Museum',
2298 'uploader_id': 'backuspagemuseum',
2299 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
2300 'upload_date': '20161008',
2301 'license': 'Standard YouTube License',
2302 'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
2303 'categories': ['Nonprofits & Activism'],
2304 'tags': list,
2305 'like_count': int,
2306 'dislike_count': int,
2307 },
2308 'params': {
2309 'noplaylist': True,
2310 'skip_download': True,
2311 },
2312 }, {
2313 'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
2314 'only_matching': True,
2315 }, {
2316 'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
2317 'only_matching': True,
2318 }, {
2319 # music album playlist
2320 'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
2321 'only_matching': True,
2322 }, {
2323 'url': 'https://invidio.us/playlist?list=PLDIoUOhQQPlXr63I_vwF9GD8sAKh77dWU',
2324 'only_matching': True,
2325 }]
2326
2327 def _real_initialize(self):
2328 self._login()
2329
2330 def _extract_mix(self, playlist_id):
2331 # The mixes are generated from a single video
2332 # the id of the playlist is just 'RD' + video_id
2333 ids = []
2334 last_id = playlist_id[-11:]
2335 for n in itertools.count(1):
2336 url = 'https://youtube.com/watch?v=%s&list=%s' % (last_id, playlist_id)
2337 webpage = self._download_webpage(
2338 url, playlist_id, 'Downloading page {0} of Youtube mix'.format(n))
2339 new_ids = orderedSet(re.findall(
2340 r'''(?xs)data-video-username=".*?".*?
2341 href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
2342 webpage))
2343 # Fetch new pages until all the videos are repeated, it seems that
2344 # there are always 51 unique videos.
2345 new_ids = [_id for _id in new_ids if _id not in ids]
2346 if not new_ids:
2347 break
2348 ids.extend(new_ids)
2349 last_id = ids[-1]
2350
2351 url_results = self._ids_to_results(ids)
2352
2353 search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
2354 title_span = (
2355 search_title('playlist-title') or
2356 search_title('title long-title') or
2357 search_title('title'))
2358 title = clean_html(title_span)
2359
2360 return self.playlist_result(url_results, playlist_id, title)
2361
2362 def _extract_playlist(self, playlist_id):
2363 url = self._TEMPLATE_URL % playlist_id
2364 page = self._download_webpage(url, playlist_id)
2365
2366 # the yt-alert-message now has tabindex attribute (see https://github.com/rg3/youtube-dl/issues/11604)
2367 for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
2368 match = match.strip()
2369 # Check if the playlist exists or is private
2370 mobj = re.match(r'[^<]*(?:The|This) playlist (?P<reason>does not exist|is private)[^<]*', match)
2371 if mobj:
2372 reason = mobj.group('reason')
2373 message = 'This playlist %s' % reason
2374 if 'private' in reason:
2375 message += ', use --username or --netrc to access it'
2376 message += '.'
2377 raise ExtractorError(message, expected=True)
2378 elif re.match(r'[^<]*Invalid parameters[^<]*', match):
2379 raise ExtractorError(
2380 'Invalid parameters. Maybe URL is incorrect.',
2381 expected=True)
2382 elif re.match(r'[^<]*Choose your language[^<]*', match):
2383 continue
2384 else:
2385 self.report_warning('Youtube gives an alert message: ' + match)
2386
2387 playlist_title = self._html_search_regex(
2388 r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
2389 page, 'title', default=None)
2390
2391 _UPLOADER_BASE = r'class=["\']pl-header-details[^>]+>\s*<li>\s*<a[^>]+\bhref='
2392 uploader = self._search_regex(
2393 r'%s["\']/(?:user|channel)/[^>]+>([^<]+)' % _UPLOADER_BASE,
2394 page, 'uploader', default=None)
2395 mobj = re.search(
2396 r'%s(["\'])(?P<path>/(?:user|channel)/(?P<uploader_id>.+?))\1' % _UPLOADER_BASE,
2397 page)
2398 if mobj:
2399 uploader_id = mobj.group('uploader_id')
2400 uploader_url = compat_urlparse.urljoin(url, mobj.group('path'))
2401 else:
2402 uploader_id = uploader_url = None
2403
2404 has_videos = True
2405
2406 if not playlist_title:
2407 try:
2408 # Some playlist URLs don't actually serve a playlist (e.g.
2409 # https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4)
2410 next(self._entries(page, playlist_id))
2411 except StopIteration:
2412 has_videos = False
2413
2414 playlist = self.playlist_result(
2415 self._entries(page, playlist_id), playlist_id, playlist_title)
2416 playlist.update({
2417 'uploader': uploader,
2418 'uploader_id': uploader_id,
2419 'uploader_url': uploader_url,
2420 })
2421
2422 return has_videos, playlist
2423
2424 def _check_download_just_video(self, url, playlist_id):
2425 # Check if it's a video-specific URL
2426 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
2427 video_id = query_dict.get('v', [None])[0] or self._search_regex(
2428 r'(?:(?:^|//)youtu\.be/|youtube\.com/embed/(?!videoseries))([0-9A-Za-z_-]{11})', url,
2429 'video id', default=None)
2430 if video_id:
2431 if self._downloader.params.get('noplaylist'):
2432 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2433 return video_id, self.url_result(video_id, 'Youtube', video_id=video_id)
2434 else:
2435 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
2436 return video_id, None
2437 return None, None
2438
2439 def _real_extract(self, url):
2440 # Extract playlist id
2441 mobj = re.match(self._VALID_URL, url)
2442 if mobj is None:
2443 raise ExtractorError('Invalid URL: %s' % url)
2444 playlist_id = mobj.group(1) or mobj.group(2)
2445
2446 video_id, video = self._check_download_just_video(url, playlist_id)
2447 if video:
2448 return video
2449
2450 if playlist_id.startswith(('RD', 'UL', 'PU')):
2451 # Mixes require a custom extraction process
2452 return self._extract_mix(playlist_id)
2453
2454 has_videos, playlist = self._extract_playlist(playlist_id)
2455 if has_videos or not video_id:
2456 return playlist
2457
2458 # Some playlist URLs don't actually serve a playlist (see
2459 # https://github.com/rg3/youtube-dl/issues/10537).
2460 # Fallback to plain video extraction if there is a video id
2461 # along with playlist id.
2462 return self.url_result(video_id, 'Youtube', video_id=video_id)
2463
2464
2465 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
2466 IE_DESC = 'YouTube.com channels'
2467 _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com|(?:www\.)?invidio\.us)/channel/(?P<id>[0-9A-Za-z_-]+)'
2468 _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
2469 _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
2470 IE_NAME = 'youtube:channel'
2471 _TESTS = [{
2472 'note': 'paginated channel',
2473 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
2474 'playlist_mincount': 91,
2475 'info_dict': {
2476 'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
2477 'title': 'Uploads from lex will',
2478 }
2479 }, {
2480 'note': 'Age restricted channel',
2481 # from https://www.youtube.com/user/DeusExOfficial
2482 'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
2483 'playlist_mincount': 64,
2484 'info_dict': {
2485 'id': 'UUs0ifCMCm1icqRbqhUINa0w',
2486 'title': 'Uploads from Deus Ex',
2487 },
2488 }, {
2489 'url': 'https://invidio.us/channel/UC23qupoDRn9YOAVzeoxjOQA',
2490 'only_matching': True,
2491 }]
2492
2493 @classmethod
2494 def suitable(cls, url):
2495 return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
2496 else super(YoutubeChannelIE, cls).suitable(url))
2497
2498 def _build_template_url(self, url, channel_id):
2499 return self._TEMPLATE_URL % channel_id
2500
2501 def _real_extract(self, url):
2502 channel_id = self._match_id(url)
2503
2504 url = self._build_template_url(url, channel_id)
2505
2506 # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
2507 # Workaround by extracting as a playlist if managed to obtain channel playlist URL
2508 # otherwise fallback on channel by page extraction
2509 channel_page = self._download_webpage(
2510 url + '?view=57', channel_id,
2511 'Downloading channel page', fatal=False)
2512 if channel_page is False:
2513 channel_playlist_id = False
2514 else:
2515 channel_playlist_id = self._html_search_meta(
2516 'channelId', channel_page, 'channel id', default=None)
2517 if not channel_playlist_id:
2518 channel_url = self._html_search_meta(
2519 ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
2520 channel_page, 'channel url', default=None)
2521 if channel_url:
2522 channel_playlist_id = self._search_regex(
2523 r'vnd\.youtube://user/([0-9A-Za-z_-]+)',
2524 channel_url, 'channel id', default=None)
2525 if channel_playlist_id and channel_playlist_id.startswith('UC'):
2526 playlist_id = 'UU' + channel_playlist_id[2:]
2527 return self.url_result(
2528 compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
2529
2530 channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
2531 autogenerated = re.search(r'''(?x)
2532 class="[^"]*?(?:
2533 channel-header-autogenerated-label|
2534 yt-channel-title-autogenerated
2535 )[^"]*"''', channel_page) is not None
2536
2537 if autogenerated:
2538 # The videos are contained in a single page
2539 # the ajax pages can't be used, they are empty
2540 entries = [
2541 self.url_result(
2542 video_id, 'Youtube', video_id=video_id,
2543 video_title=video_title)
2544 for video_id, video_title in self.extract_videos_from_page(channel_page)]
2545 return self.playlist_result(entries, channel_id)
2546
2547 try:
2548 next(self._entries(channel_page, channel_id))
2549 except StopIteration:
2550 alert_message = self._html_search_regex(
2551 r'(?s)<div[^>]+class=(["\']).*?\byt-alert-message\b.*?\1[^>]*>(?P<alert>[^<]+)</div>',
2552 channel_page, 'alert', default=None, group='alert')
2553 if alert_message:
2554 raise ExtractorError('Youtube said: %s' % alert_message, expected=True)
2555
2556 return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
2557
2558
2559 class YoutubeUserIE(YoutubeChannelIE):
2560 IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
2561 _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:(?P<user>user|c)/)?(?!(?:attribution_link|watch|results|shared)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
2562 _TEMPLATE_URL = 'https://www.youtube.com/%s/%s/videos'
2563 IE_NAME = 'youtube:user'
2564
2565 _TESTS = [{
2566 'url': 'https://www.youtube.com/user/TheLinuxFoundation',
2567 'playlist_mincount': 320,
2568 'info_dict': {
2569 'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
2570 'title': 'Uploads from The Linux Foundation',
2571 }
2572 }, {
2573 # Only available via https://www.youtube.com/c/12minuteathlete/videos
2574 # but not https://www.youtube.com/user/12minuteathlete/videos
2575 'url': 'https://www.youtube.com/c/12minuteathlete/videos',
2576 'playlist_mincount': 249,
2577 'info_dict': {
2578 'id': 'UUVjM-zV6_opMDx7WYxnjZiQ',
2579 'title': 'Uploads from 12 Minute Athlete',
2580 }
2581 }, {
2582 'url': 'ytuser:phihag',
2583 'only_matching': True,
2584 }, {
2585 'url': 'https://www.youtube.com/c/gametrailers',
2586 'only_matching': True,
2587 }, {
2588 'url': 'https://www.youtube.com/gametrailers',
2589 'only_matching': True,
2590 }, {
2591 # This channel is not available, geo restricted to JP
2592 'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
2593 'only_matching': True,
2594 }]
2595
2596 @classmethod
2597 def suitable(cls, url):
2598 # Don't return True if the url can be extracted with other youtube
2599 # extractor, the regex would is too permissive and it would match.
2600 other_yt_ies = iter(klass for (name, klass) in globals().items() if name.startswith('Youtube') and name.endswith('IE') and klass is not cls)
2601 if any(ie.suitable(url) for ie in other_yt_ies):
2602 return False
2603 else:
2604 return super(YoutubeUserIE, cls).suitable(url)
2605
2606 def _build_template_url(self, url, channel_id):
2607 mobj = re.match(self._VALID_URL, url)
2608 return self._TEMPLATE_URL % (mobj.group('user') or 'user', mobj.group('id'))
2609
2610
2611 class YoutubeLiveIE(YoutubeBaseInfoExtractor):
2612 IE_DESC = 'YouTube.com live streams'
2613 _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:(?:user|channel|c)/)?(?P<id>[^/]+))/live'
2614 IE_NAME = 'youtube:live'
2615
2616 _TESTS = [{
2617 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
2618 'info_dict': {
2619 'id': 'a48o2S1cPoo',
2620 'ext': 'mp4',
2621 'title': 'The Young Turks - Live Main Show',
2622 'uploader': 'The Young Turks',
2623 'uploader_id': 'TheYoungTurks',
2624 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
2625 'upload_date': '20150715',
2626 'license': 'Standard YouTube License',
2627 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
2628 'categories': ['News & Politics'],
2629 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
2630 'like_count': int,
2631 'dislike_count': int,
2632 },
2633 'params': {
2634 'skip_download': True,
2635 },
2636 }, {
2637 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
2638 'only_matching': True,
2639 }, {
2640 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
2641 'only_matching': True,
2642 }, {
2643 'url': 'https://www.youtube.com/TheYoungTurks/live',
2644 'only_matching': True,
2645 }]
2646
2647 def _real_extract(self, url):
2648 mobj = re.match(self._VALID_URL, url)
2649 channel_id = mobj.group('id')
2650 base_url = mobj.group('base_url')
2651 webpage = self._download_webpage(url, channel_id, fatal=False)
2652 if webpage:
2653 page_type = self._og_search_property(
2654 'type', webpage, 'page type', default='')
2655 video_id = self._html_search_meta(
2656 'videoId', webpage, 'video id', default=None)
2657 if page_type.startswith('video') and video_id and re.match(
2658 r'^[0-9A-Za-z_-]{11}$', video_id):
2659 return self.url_result(video_id, YoutubeIE.ie_key())
2660 return self.url_result(base_url)
2661
2662
2663 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
2664 IE_DESC = 'YouTube.com user/channel playlists'
2665 _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
2666 IE_NAME = 'youtube:playlists'
2667
2668 _TESTS = [{
2669 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
2670 'playlist_mincount': 4,
2671 'info_dict': {
2672 'id': 'ThirstForScience',
2673 'title': 'Thirst for Science',
2674 },
2675 }, {
2676 # with "Load more" button
2677 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
2678 'playlist_mincount': 70,
2679 'info_dict': {
2680 'id': 'igorkle1',
2681 'title': 'Š˜Š³Š¾Ń€ŃŒ ŠšŠ»ŠµŠ¹Š½ŠµŃ€',
2682 },
2683 }, {
2684 'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
2685 'playlist_mincount': 17,
2686 'info_dict': {
2687 'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
2688 'title': 'Chem Player',
2689 },
2690 }]
2691
2692
2693 class YoutubeSearchBaseInfoExtractor(YoutubePlaylistBaseInfoExtractor):
2694 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
2695
2696
2697 class YoutubeSearchIE(SearchInfoExtractor, YoutubeSearchBaseInfoExtractor):
2698 IE_DESC = 'YouTube.com searches'
2699 # there doesn't appear to be a real limit, for example if you search for
2700 # 'python' you get more than 8.000.000 results
2701 _MAX_RESULTS = float('inf')
2702 IE_NAME = 'youtube:search'
2703 _SEARCH_KEY = 'ytsearch'
2704 _EXTRA_QUERY_ARGS = {}
2705 _TESTS = []
2706
2707 def _get_n_results(self, query, n):
2708 """Get a specified number of results for a query"""
2709
2710 videos = []
2711 limit = n
2712
2713 url_query = {
2714 'search_query': query.encode('utf-8'),
2715 }
2716 url_query.update(self._EXTRA_QUERY_ARGS)
2717 result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
2718
2719 for pagenum in itertools.count(1):
2720 data = self._download_json(
2721 result_url, video_id='query "%s"' % query,
2722 note='Downloading page %s' % pagenum,
2723 errnote='Unable to download API page',
2724 query={'spf': 'navigate'})
2725 html_content = data[1]['body']['content']
2726
2727 if 'class="search-message' in html_content:
2728 raise ExtractorError(
2729 '[youtube] No video results', expected=True)
2730
2731 new_videos = list(self._process_page(html_content))
2732 videos += new_videos
2733 if not new_videos or len(videos) > limit:
2734 break
2735 next_link = self._html_search_regex(
2736 r'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*<span[^>]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next',
2737 html_content, 'next link', default=None)
2738 if next_link is None:
2739 break
2740 result_url = compat_urlparse.urljoin('https://www.youtube.com/', next_link)
2741
2742 if len(videos) > n:
2743 videos = videos[:n]
2744 return self.playlist_result(videos, query)
2745
2746
2747 class YoutubeSearchDateIE(YoutubeSearchIE):
2748 IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
2749 _SEARCH_KEY = 'ytsearchdate'
2750 IE_DESC = 'YouTube.com searches, newest videos first'
2751 _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
2752
2753
2754 class YoutubeSearchURLIE(YoutubeSearchBaseInfoExtractor):
2755 IE_DESC = 'YouTube.com search URLs'
2756 IE_NAME = 'youtube:search_url'
2757 _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
2758 _TESTS = [{
2759 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
2760 'playlist_mincount': 5,
2761 'info_dict': {
2762 'title': 'youtube-dl test video',
2763 }
2764 }, {
2765 'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
2766 'only_matching': True,
2767 }]
2768
2769 def _real_extract(self, url):
2770 mobj = re.match(self._VALID_URL, url)
2771 query = compat_urllib_parse_unquote_plus(mobj.group('query'))
2772 webpage = self._download_webpage(url, query)
2773 return self.playlist_result(self._process_page(webpage), playlist_title=query)
2774
2775
2776 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
2777 IE_DESC = 'YouTube.com (multi-season) shows'
2778 _VALID_URL = r'https?://(?:www\.)?youtube\.com/show/(?P<id>[^?#]*)'
2779 IE_NAME = 'youtube:show'
2780 _TESTS = [{
2781 'url': 'https://www.youtube.com/show/airdisasters',
2782 'playlist_mincount': 5,
2783 'info_dict': {
2784 'id': 'airdisasters',
2785 'title': 'Air Disasters',
2786 }
2787 }]
2788
2789 def _real_extract(self, url):
2790 playlist_id = self._match_id(url)
2791 return super(YoutubeShowIE, self)._real_extract(
2792 'https://www.youtube.com/show/%s/playlists' % playlist_id)
2793
2794
2795 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
2796 """
2797 Base class for feed extractors
2798 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
2799 """
2800 _LOGIN_REQUIRED = True
2801
2802 @property
2803 def IE_NAME(self):
2804 return 'youtube:%s' % self._FEED_NAME
2805
2806 def _real_initialize(self):
2807 self._login()
2808
2809 def _entries(self, page):
2810 # The extraction process is the same as for playlists, but the regex
2811 # for the video ids doesn't contain an index
2812 ids = []
2813 more_widget_html = content_html = page
2814 for page_num in itertools.count(1):
2815 matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
2816
2817 # 'recommended' feed has infinite 'load more' and each new portion spins
2818 # the same videos in (sometimes) slightly different order, so we'll check
2819 # for unicity and break when portion has no new videos
2820 new_ids = list(filter(lambda video_id: video_id not in ids, orderedSet(matches)))
2821 if not new_ids:
2822 break
2823
2824 ids.extend(new_ids)
2825
2826 for entry in self._ids_to_results(new_ids):
2827 yield entry
2828
2829 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
2830 if not mobj:
2831 break
2832
2833 more = self._download_json(
2834 'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
2835 'Downloading page #%s' % page_num,
2836 transform_source=uppercase_escape)
2837 content_html = more['content_html']
2838 more_widget_html = more['load_more_widget_html']
2839
2840 def _real_extract(self, url):
2841 page = self._download_webpage(
2842 'https://www.youtube.com/feed/%s' % self._FEED_NAME,
2843 self._PLAYLIST_TITLE)
2844 return self.playlist_result(
2845 self._entries(page), playlist_title=self._PLAYLIST_TITLE)
2846
2847
2848 class YoutubeWatchLaterIE(YoutubePlaylistIE):
2849 IE_NAME = 'youtube:watchlater'
2850 IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
2851 _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
2852
2853 _TESTS = [{
2854 'url': 'https://www.youtube.com/playlist?list=WL',
2855 'only_matching': True,
2856 }, {
2857 'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
2858 'only_matching': True,
2859 }]
2860
2861 def _real_extract(self, url):
2862 _, video = self._check_download_just_video(url, 'WL')
2863 if video:
2864 return video
2865 _, playlist = self._extract_playlist('WL')
2866 return playlist
2867
2868
2869 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
2870 IE_NAME = 'youtube:favorites'
2871 IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
2872 _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
2873 _LOGIN_REQUIRED = True
2874
2875 def _real_extract(self, url):
2876 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
2877 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
2878 return self.url_result(playlist_id, 'YoutubePlaylist')
2879
2880
2881 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
2882 IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
2883 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/recommended|:ytrec(?:ommended)?'
2884 _FEED_NAME = 'recommended'
2885 _PLAYLIST_TITLE = 'Youtube Recommended videos'
2886
2887
2888 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
2889 IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
2890 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
2891 _FEED_NAME = 'subscriptions'
2892 _PLAYLIST_TITLE = 'Youtube Subscriptions'
2893
2894
2895 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
2896 IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
2897 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
2898 _FEED_NAME = 'history'
2899 _PLAYLIST_TITLE = 'Youtube History'
2900
2901
2902 class YoutubeTruncatedURLIE(InfoExtractor):
2903 IE_NAME = 'youtube:truncated_url'
2904 IE_DESC = False # Do not list
2905 _VALID_URL = r'''(?x)
2906 (?:https?://)?
2907 (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
2908 (?:watch\?(?:
2909 feature=[a-z_]+|
2910 annotation_id=annotation_[^&]+|
2911 x-yt-cl=[0-9]+|
2912 hl=[^&]*|
2913 t=[0-9]+
2914 )?
2915 |
2916 attribution_link\?a=[^&]+
2917 )
2918 $
2919 '''
2920
2921 _TESTS = [{
2922 'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
2923 'only_matching': True,
2924 }, {
2925 'url': 'https://www.youtube.com/watch?',
2926 'only_matching': True,
2927 }, {
2928 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
2929 'only_matching': True,
2930 }, {
2931 'url': 'https://www.youtube.com/watch?feature=foo',
2932 'only_matching': True,
2933 }, {
2934 'url': 'https://www.youtube.com/watch?hl=en-GB',
2935 'only_matching': True,
2936 }, {
2937 'url': 'https://www.youtube.com/watch?t=2372',
2938 'only_matching': True,
2939 }]
2940
2941 def _real_extract(self, url):
2942 raise ExtractorError(
2943 'Did you forget to quote the URL? Remember that & is a meta '
2944 'character in most shells, so you want to put the URL in quotes, '
2945 'like youtube-dl '
2946 '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
2947 ' or simply youtube-dl BaW_jenozKc .',
2948 expected=True)
2949
2950
2951 class YoutubeTruncatedIDIE(InfoExtractor):
2952 IE_NAME = 'youtube:truncated_id'
2953 IE_DESC = False # Do not list
2954 _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
2955
2956 _TESTS = [{
2957 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
2958 'only_matching': True,
2959 }]
2960
2961 def _real_extract(self, url):
2962 video_id = self._match_id(url)
2963 raise ExtractorError(
2964 'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
2965 expected=True)