]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/youtube.py
dec02804bf52770719f7f29b7fe893f22b26cc5f
[youtubedl] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 from __future__ import unicode_literals
4
5
6 import itertools
7 import json
8 import os.path
9 import random
10 import re
11 import time
12 import traceback
13
14 from .common import InfoExtractor, SearchInfoExtractor
15 from ..jsinterp import JSInterpreter
16 from ..swfinterp import SWFInterpreter
17 from ..compat import (
18 compat_chr,
19 compat_parse_qs,
20 compat_urllib_parse_unquote,
21 compat_urllib_parse_unquote_plus,
22 compat_urllib_parse_urlencode,
23 compat_urllib_parse_urlparse,
24 compat_urlparse,
25 compat_str,
26 )
27 from ..utils import (
28 clean_html,
29 error_to_compat_str,
30 ExtractorError,
31 float_or_none,
32 get_element_by_attribute,
33 get_element_by_id,
34 int_or_none,
35 mimetype2ext,
36 orderedSet,
37 parse_codecs,
38 parse_duration,
39 remove_quotes,
40 remove_start,
41 sanitized_Request,
42 smuggle_url,
43 str_to_int,
44 try_get,
45 unescapeHTML,
46 unified_strdate,
47 unsmuggle_url,
48 uppercase_escape,
49 urlencode_postdata,
50 ISO3166Utils,
51 )
52
53
54 class YoutubeBaseInfoExtractor(InfoExtractor):
55 """Provide base functions for Youtube extractors"""
56 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
57 _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
58 _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/signin/challenge/sl/password'
59 _NETRC_MACHINE = 'youtube'
60 # If True it will raise an error if no login info is provided
61 _LOGIN_REQUIRED = False
62
63 def _set_language(self):
64 self._set_cookie(
65 '.youtube.com', 'PREF', 'f1=50000000&hl=en',
66 # YouTube sets the expire time to about two months
67 expire_time=time.time() + 2 * 30 * 24 * 3600)
68
69 def _ids_to_results(self, ids):
70 return [
71 self.url_result(vid_id, 'Youtube', video_id=vid_id)
72 for vid_id in ids]
73
74 def _login(self):
75 """
76 Attempt to log in to YouTube.
77 True is returned if successful or skipped.
78 False is returned if login failed.
79
80 If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
81 """
82 (username, password) = self._get_login_info()
83 # No authentication to be performed
84 if username is None:
85 if self._LOGIN_REQUIRED:
86 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
87 return True
88
89 login_page = self._download_webpage(
90 self._LOGIN_URL, None,
91 note='Downloading login page',
92 errnote='unable to fetch login page', fatal=False)
93 if login_page is False:
94 return
95
96 login_form = self._hidden_inputs(login_page)
97
98 login_form.update({
99 'checkConnection': 'youtube',
100 'Email': username,
101 'Passwd': password,
102 })
103
104 login_results = self._download_webpage(
105 self._PASSWORD_CHALLENGE_URL, None,
106 note='Logging in', errnote='unable to log in', fatal=False,
107 data=urlencode_postdata(login_form))
108 if login_results is False:
109 return False
110
111 error_msg = self._html_search_regex(
112 r'<[^>]+id="errormsg_0_Passwd"[^>]*>([^<]+)<',
113 login_results, 'error message', default=None)
114 if error_msg:
115 raise ExtractorError('Unable to login: %s' % error_msg, expected=True)
116
117 if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
118 raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
119
120 # Two-Factor
121 # TODO add SMS and phone call support - these require making a request and then prompting the user
122
123 if re.search(r'(?i)<form[^>]+id="challenge"', login_results) is not None:
124 tfa_code = self._get_tfa_info('2-step verification code')
125
126 if not tfa_code:
127 self._downloader.report_warning(
128 'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
129 '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
130 return False
131
132 tfa_code = remove_start(tfa_code, 'G-')
133
134 tfa_form_strs = self._form_hidden_inputs('challenge', login_results)
135
136 tfa_form_strs.update({
137 'Pin': tfa_code,
138 'TrustDevice': 'on',
139 })
140
141 tfa_data = urlencode_postdata(tfa_form_strs)
142
143 tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
144 tfa_results = self._download_webpage(
145 tfa_req, None,
146 note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
147
148 if tfa_results is False:
149 return False
150
151 if re.search(r'(?i)<form[^>]+id="challenge"', tfa_results) is not None:
152 self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
153 return False
154 if re.search(r'(?i)<form[^>]+id="gaia_loginform"', tfa_results) is not None:
155 self._downloader.report_warning('unable to log in - did the page structure change?')
156 return False
157 if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
158 self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
159 return False
160
161 if re.search(r'(?i)<form[^>]+id="gaia_loginform"', login_results) is not None:
162 self._downloader.report_warning('unable to log in: bad username or password')
163 return False
164 return True
165
166 def _real_initialize(self):
167 if self._downloader is None:
168 return
169 self._set_language()
170 if not self._login():
171 return
172
173
174 class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
175 # Extract entries from page with "Load more" button
176 def _entries(self, page, playlist_id):
177 more_widget_html = content_html = page
178 for page_num in itertools.count(1):
179 for entry in self._process_page(content_html):
180 yield entry
181
182 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
183 if not mobj:
184 break
185
186 more = self._download_json(
187 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
188 'Downloading page #%s' % page_num,
189 transform_source=uppercase_escape)
190 content_html = more['content_html']
191 if not content_html.strip():
192 # Some webpages show a "Load more" button but they don't
193 # have more videos
194 break
195 more_widget_html = more['load_more_widget_html']
196
197
198 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
199 def _process_page(self, content):
200 for video_id, video_title in self.extract_videos_from_page(content):
201 yield self.url_result(video_id, 'Youtube', video_id, video_title)
202
203 def extract_videos_from_page(self, page):
204 ids_in_page = []
205 titles_in_page = []
206 for mobj in re.finditer(self._VIDEO_RE, page):
207 # The link with index 0 is not the first video of the playlist (not sure if still actual)
208 if 'index' in mobj.groupdict() and mobj.group('id') == '0':
209 continue
210 video_id = mobj.group('id')
211 video_title = unescapeHTML(mobj.group('title'))
212 if video_title:
213 video_title = video_title.strip()
214 try:
215 idx = ids_in_page.index(video_id)
216 if video_title and not titles_in_page[idx]:
217 titles_in_page[idx] = video_title
218 except ValueError:
219 ids_in_page.append(video_id)
220 titles_in_page.append(video_title)
221 return zip(ids_in_page, titles_in_page)
222
223
224 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
225 def _process_page(self, content):
226 for playlist_id in orderedSet(re.findall(
227 r'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
228 content)):
229 yield self.url_result(
230 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
231
232 def _real_extract(self, url):
233 playlist_id = self._match_id(url)
234 webpage = self._download_webpage(url, playlist_id)
235 title = self._og_search_title(webpage, fatal=False)
236 return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
237
238
239 class YoutubeIE(YoutubeBaseInfoExtractor):
240 IE_DESC = 'YouTube.com'
241 _VALID_URL = r"""(?x)^
242 (
243 (?:https?://|//) # http(s):// or protocol-independent URL
244 (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
245 (?:www\.)?deturl\.com/www\.youtube\.com/|
246 (?:www\.)?pwnyoutube\.com/|
247 (?:www\.)?yourepeat\.com/|
248 tube\.majestyc\.net/|
249 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
250 (?:.*?\#/)? # handle anchor (#/) redirect urls
251 (?: # the various things that can precede the ID:
252 (?:(?:v|embed|e)/(?!videoseries)) # v/ or embed/ or e/
253 |(?: # or the v= param in all its forms
254 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
255 (?:\?|\#!?) # the params delimiter ? or # or #!
256 (?:.*?[&;])?? # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
257 v=
258 )
259 ))
260 |(?:
261 youtu\.be| # just youtu.be/xxxx
262 vid\.plus| # or vid.plus/xxxx
263 zwearz\.com/watch| # or zwearz.com/watch/xxxx
264 )/
265 |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
266 )
267 )? # all until now is optional -> you can pass the naked ID
268 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
269 (?!.*?\blist=) # combined list/video URLs are handled by the playlist IE
270 (?(1).+)? # if we found the ID, everything can follow
271 $"""
272 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
273 _formats = {
274 '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
275 '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
276 '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
277 '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
278 '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
279 '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
280 '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
281 '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
282 # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
283 '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
284 '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
285 '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
286 '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
287 '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
288 '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
289 '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
290 '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
291 '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
292
293
294 # 3D videos
295 '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
296 '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
297 '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
298 '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
299 '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
300 '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
301 '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
302
303 # Apple HTTP Live Streaming
304 '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
305 '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
306 '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
307 '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
308 '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
309 '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
310 '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
311 '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
312
313 # DASH mp4 video
314 '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
315 '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
316 '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
317 '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
318 '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
319 '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}, # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
320 '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
321 '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
322 '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
323 '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
324 '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
325 '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
326
327 # Dash mp4 audio
328 '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
329 '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
330 '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
331 '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
332 '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
333 '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'preference': -50, 'container': 'm4a_dash'},
334 '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'preference': -50, 'container': 'm4a_dash'},
335
336 # Dash webm
337 '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
338 '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
339 '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
340 '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
341 '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
342 '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
343 '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
344 '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
345 '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
346 '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
347 '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
348 '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
349 '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
350 '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
351 '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
352 # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
353 '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
354 '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
355 '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
356 '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
357 '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
358 '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
359
360 # Dash webm audio
361 '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
362 '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
363
364 # Dash webm audio with opus inside
365 '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
366 '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
367 '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
368
369 # RTMP (unnamed)
370 '_rtmp': {'protocol': 'rtmp'},
371 }
372 _SUBTITLE_FORMATS = ('ttml', 'vtt')
373
374 IE_NAME = 'youtube'
375 _TESTS = [
376 {
377 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
378 'info_dict': {
379 'id': 'BaW_jenozKc',
380 'ext': 'mp4',
381 'title': 'youtube-dl test video "\'/\\Ƥā†­š•',
382 'uploader': 'Philipp Hagemeister',
383 'uploader_id': 'phihag',
384 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
385 'upload_date': '20121002',
386 'license': 'Standard YouTube License',
387 'description': 'test chars: "\'/\\Ƥā†­š•\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
388 'categories': ['Science & Technology'],
389 'tags': ['youtube-dl'],
390 'duration': 10,
391 'like_count': int,
392 'dislike_count': int,
393 'start_time': 1,
394 'end_time': 9,
395 }
396 },
397 {
398 'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY',
399 'note': 'Test generic use_cipher_signature video (#897)',
400 'info_dict': {
401 'id': 'UxxajLWwzqY',
402 'ext': 'mp4',
403 'upload_date': '20120506',
404 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
405 'alt_title': 'I Love It (feat. Charli XCX)',
406 'description': 'md5:f3ceb5ef83a08d95b9d146f973157cc8',
407 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
408 'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
409 'iconic ep', 'iconic', 'love', 'it'],
410 'duration': 180,
411 'uploader': 'Icona Pop',
412 'uploader_id': 'IconaPop',
413 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop',
414 'license': 'Standard YouTube License',
415 'creator': 'Icona Pop',
416 }
417 },
418 {
419 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
420 'note': 'Test VEVO video with age protection (#956)',
421 'info_dict': {
422 'id': '07FYdnEawAQ',
423 'ext': 'mp4',
424 'upload_date': '20130703',
425 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
426 'alt_title': 'Tunnel Vision',
427 'description': 'md5:64249768eec3bc4276236606ea996373',
428 'duration': 419,
429 'uploader': 'justintimberlakeVEVO',
430 'uploader_id': 'justintimberlakeVEVO',
431 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
432 'license': 'Standard YouTube License',
433 'creator': 'Justin Timberlake',
434 'age_limit': 18,
435 }
436 },
437 {
438 'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
439 'note': 'Embed-only video (#1746)',
440 'info_dict': {
441 'id': 'yZIXLfi8CZQ',
442 'ext': 'mp4',
443 'upload_date': '20120608',
444 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
445 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
446 'uploader': 'SET India',
447 'uploader_id': 'setindia',
448 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
449 'license': 'Standard YouTube License',
450 'age_limit': 18,
451 }
452 },
453 {
454 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
455 'note': 'Use the first video ID in the URL',
456 'info_dict': {
457 'id': 'BaW_jenozKc',
458 'ext': 'mp4',
459 'title': 'youtube-dl test video "\'/\\Ƥā†­š•',
460 'uploader': 'Philipp Hagemeister',
461 'uploader_id': 'phihag',
462 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
463 'upload_date': '20121002',
464 'license': 'Standard YouTube License',
465 'description': 'test chars: "\'/\\Ƥā†­š•\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
466 'categories': ['Science & Technology'],
467 'tags': ['youtube-dl'],
468 'duration': 10,
469 'like_count': int,
470 'dislike_count': int,
471 },
472 'params': {
473 'skip_download': True,
474 },
475 },
476 {
477 'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
478 'note': '256k DASH audio (format 141) via DASH manifest',
479 'info_dict': {
480 'id': 'a9LDPn-MO4I',
481 'ext': 'm4a',
482 'upload_date': '20121002',
483 'uploader_id': '8KVIDEO',
484 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
485 'description': '',
486 'uploader': '8KVIDEO',
487 'license': 'Standard YouTube License',
488 'title': 'UHDTV TEST 8K VIDEO.mp4'
489 },
490 'params': {
491 'youtube_include_dash_manifest': True,
492 'format': '141',
493 },
494 'skip': 'format 141 not served anymore',
495 },
496 # DASH manifest with encrypted signature
497 {
498 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
499 'info_dict': {
500 'id': 'IB3lcPjvWLA',
501 'ext': 'm4a',
502 'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
503 'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
504 'duration': 244,
505 'uploader': 'AfrojackVEVO',
506 'uploader_id': 'AfrojackVEVO',
507 'upload_date': '20131011',
508 'license': 'Standard YouTube License',
509 },
510 'params': {
511 'youtube_include_dash_manifest': True,
512 'format': '141/bestaudio[ext=m4a]',
513 },
514 },
515 # JS player signature function name containing $
516 {
517 'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
518 'info_dict': {
519 'id': 'nfWlot6h_JM',
520 'ext': 'm4a',
521 'title': 'Taylor Swift - Shake It Off',
522 'alt_title': 'Shake It Off',
523 'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
524 'duration': 242,
525 'uploader': 'TaylorSwiftVEVO',
526 'uploader_id': 'TaylorSwiftVEVO',
527 'upload_date': '20140818',
528 'license': 'Standard YouTube License',
529 'creator': 'Taylor Swift',
530 },
531 'params': {
532 'youtube_include_dash_manifest': True,
533 'format': '141/bestaudio[ext=m4a]',
534 },
535 },
536 # Controversy video
537 {
538 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
539 'info_dict': {
540 'id': 'T4XJQO3qol8',
541 'ext': 'mp4',
542 'duration': 219,
543 'upload_date': '20100909',
544 'uploader': 'The Amazing Atheist',
545 'uploader_id': 'TheAmazingAtheist',
546 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
547 'license': 'Standard YouTube License',
548 'title': 'Burning Everyone\'s Koran',
549 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
550 }
551 },
552 # Normal age-gate video (No vevo, embed allowed)
553 {
554 'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
555 'info_dict': {
556 'id': 'HtVdAasjOgU',
557 'ext': 'mp4',
558 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
559 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
560 'duration': 142,
561 'uploader': 'The Witcher',
562 'uploader_id': 'WitcherGame',
563 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
564 'upload_date': '20140605',
565 'license': 'Standard YouTube License',
566 'age_limit': 18,
567 },
568 },
569 # Age-gate video with encrypted signature
570 {
571 'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
572 'info_dict': {
573 'id': '6kLq3WMV1nU',
574 'ext': 'mp4',
575 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
576 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
577 'duration': 247,
578 'uploader': 'LloydVEVO',
579 'uploader_id': 'LloydVEVO',
580 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
581 'upload_date': '20110629',
582 'license': 'Standard YouTube License',
583 'age_limit': 18,
584 },
585 },
586 # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
587 {
588 'url': '__2ABJjxzNo',
589 'info_dict': {
590 'id': '__2ABJjxzNo',
591 'ext': 'mp4',
592 'duration': 266,
593 'upload_date': '20100430',
594 'uploader_id': 'deadmau5',
595 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
596 'creator': 'deadmau5',
597 'description': 'md5:12c56784b8032162bb936a5f76d55360',
598 'uploader': 'deadmau5',
599 'license': 'Standard YouTube License',
600 'title': 'Deadmau5 - Some Chords (HD)',
601 'alt_title': 'Some Chords',
602 },
603 'expected_warnings': [
604 'DASH manifest missing',
605 ]
606 },
607 # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
608 {
609 'url': 'lqQg6PlCWgI',
610 'info_dict': {
611 'id': 'lqQg6PlCWgI',
612 'ext': 'mp4',
613 'duration': 6085,
614 'upload_date': '20150827',
615 'uploader_id': 'olympic',
616 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
617 'license': 'Standard YouTube License',
618 'description': 'HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
619 'uploader': 'Olympic',
620 'title': 'Hockey - Women - GER-AUS - London 2012 Olympic Games',
621 },
622 'params': {
623 'skip_download': 'requires avconv',
624 }
625 },
626 # Non-square pixels
627 {
628 'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
629 'info_dict': {
630 'id': '_b-2C3KPAM0',
631 'ext': 'mp4',
632 'stretched_ratio': 16 / 9.,
633 'duration': 85,
634 'upload_date': '20110310',
635 'uploader_id': 'AllenMeow',
636 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
637 'description': 'made by Wacom from Korea | 字幕&åŠ ę²¹ę·»é†‹ by TY\'s Allen | ę„Ÿč¬heylisa00cavey1001同å­øē†±ęƒ…ęä¾›ę¢—åŠēæ»č­Æ',
638 'uploader': 'å­«č‰¾å€«',
639 'license': 'Standard YouTube License',
640 'title': '[A-made] č®Šę…‹å¦å­—å¹•ē‰ˆ å¤Ŗ妍 ęˆ‘å°±ę˜Æ這ęØ£ēš„äŗŗ',
641 },
642 },
643 # url_encoded_fmt_stream_map is empty string
644 {
645 'url': 'qEJwOuvDf7I',
646 'info_dict': {
647 'id': 'qEJwOuvDf7I',
648 'ext': 'webm',
649 'title': 'ŠžŠ±ŃŃƒŠ¶Š“ŠµŠ½ŠøŠµ суŠ“ŠµŠ±Š½Š¾Š¹ ŠæрŠ°ŠŗтŠøŠŗŠø ŠæŠ¾ Š²Ń‹Š±Š¾Ń€Š°Š¼ 14 сŠµŠ½Ń‚яŠ±Ń€Ń 2014 Š³Š¾Š“Š° Š² Š”Š°Š½Šŗт-ŠŸŠµŃ‚ŠµŃ€Š±ŃƒŃ€Š³Šµ',
650 'description': '',
651 'upload_date': '20150404',
652 'uploader_id': 'spbelect',
653 'uploader': 'ŠŠ°Š±Š»ŃŽŠ“Š°Ń‚ŠµŠ»Šø ŠŸŠµŃ‚ŠµŃ€Š±ŃƒŃ€Š³Š°',
654 },
655 'params': {
656 'skip_download': 'requires avconv',
657 },
658 'skip': 'This live event has ended.',
659 },
660 # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
661 {
662 'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
663 'info_dict': {
664 'id': 'FIl7x6_3R5Y',
665 'ext': 'mp4',
666 'title': 'md5:7b81415841e02ecd4313668cde88737a',
667 'description': 'md5:116377fd2963b81ec4ce64b542173306',
668 'duration': 220,
669 'upload_date': '20150625',
670 'uploader_id': 'dorappi2000',
671 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
672 'uploader': 'dorappi2000',
673 'license': 'Standard YouTube License',
674 'formats': 'mincount:32',
675 },
676 },
677 # DASH manifest with segment_list
678 {
679 'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
680 'md5': '8ce563a1d667b599d21064e982ab9e31',
681 'info_dict': {
682 'id': 'CsmdDsKjzN8',
683 'ext': 'mp4',
684 'upload_date': '20150501', # According to '<meta itemprop="datePublished"', but in other places it's 20150510
685 'uploader': 'Airtek',
686 'description': 'RetransmisiĆ³n en directo de la XVIII media maratĆ³n de Zaragoza.',
687 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
688 'license': 'Standard YouTube License',
689 'title': 'RetransmisiĆ³n XVIII Media maratĆ³n Zaragoza 2015',
690 },
691 'params': {
692 'youtube_include_dash_manifest': True,
693 'format': '135', # bestvideo
694 },
695 'skip': 'This live event has ended.',
696 },
697 {
698 # Multifeed videos (multiple cameras), URL is for Main Camera
699 'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
700 'info_dict': {
701 'id': 'jqWvoWXjCVs',
702 'title': 'teamPGP: Rocket League Noob Stream',
703 'description': 'md5:dc7872fb300e143831327f1bae3af010',
704 },
705 'playlist': [{
706 'info_dict': {
707 'id': 'jqWvoWXjCVs',
708 'ext': 'mp4',
709 'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
710 'description': 'md5:dc7872fb300e143831327f1bae3af010',
711 'duration': 7335,
712 'upload_date': '20150721',
713 'uploader': 'Beer Games Beer',
714 'uploader_id': 'beergamesbeer',
715 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
716 'license': 'Standard YouTube License',
717 },
718 }, {
719 'info_dict': {
720 'id': '6h8e8xoXJzg',
721 'ext': 'mp4',
722 'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
723 'description': 'md5:dc7872fb300e143831327f1bae3af010',
724 'duration': 7337,
725 'upload_date': '20150721',
726 'uploader': 'Beer Games Beer',
727 'uploader_id': 'beergamesbeer',
728 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
729 'license': 'Standard YouTube License',
730 },
731 }, {
732 'info_dict': {
733 'id': 'PUOgX5z9xZw',
734 'ext': 'mp4',
735 'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
736 'description': 'md5:dc7872fb300e143831327f1bae3af010',
737 'duration': 7337,
738 'upload_date': '20150721',
739 'uploader': 'Beer Games Beer',
740 'uploader_id': 'beergamesbeer',
741 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
742 'license': 'Standard YouTube License',
743 },
744 }, {
745 'info_dict': {
746 'id': 'teuwxikvS5k',
747 'ext': 'mp4',
748 'title': 'teamPGP: Rocket League Noob Stream (zim)',
749 'description': 'md5:dc7872fb300e143831327f1bae3af010',
750 'duration': 7334,
751 'upload_date': '20150721',
752 'uploader': 'Beer Games Beer',
753 'uploader_id': 'beergamesbeer',
754 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
755 'license': 'Standard YouTube License',
756 },
757 }],
758 'params': {
759 'skip_download': True,
760 },
761 },
762 {
763 # Multifeed video with comma in title (see https://github.com/rg3/youtube-dl/issues/8536)
764 'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
765 'info_dict': {
766 'id': 'gVfLd0zydlo',
767 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
768 },
769 'playlist_count': 2,
770 'skip': 'Not multifeed anymore',
771 },
772 {
773 'url': 'https://vid.plus/FlRa-iH7PGw',
774 'only_matching': True,
775 },
776 {
777 'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
778 'only_matching': True,
779 },
780 {
781 # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
782 # Also tests cut-off URL expansion in video description (see
783 # https://github.com/rg3/youtube-dl/issues/1892,
784 # https://github.com/rg3/youtube-dl/issues/8164)
785 'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
786 'info_dict': {
787 'id': 'lsguqyKfVQg',
788 'ext': 'mp4',
789 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
790 'alt_title': 'Dark Walk',
791 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
792 'duration': 133,
793 'upload_date': '20151119',
794 'uploader_id': 'IronSoulElf',
795 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
796 'uploader': 'IronSoulElf',
797 'license': 'Standard YouTube License',
798 'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
799 },
800 'params': {
801 'skip_download': True,
802 },
803 },
804 {
805 # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
806 'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
807 'only_matching': True,
808 },
809 {
810 # Video with yt:stretch=17:0
811 'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
812 'info_dict': {
813 'id': 'Q39EVAstoRM',
814 'ext': 'mp4',
815 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
816 'description': 'md5:ee18a25c350637c8faff806845bddee9',
817 'upload_date': '20151107',
818 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
819 'uploader': 'CH GAMER DROID',
820 },
821 'params': {
822 'skip_download': True,
823 },
824 'skip': 'This video does not exist.',
825 },
826 {
827 # Video licensed under Creative Commons
828 'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
829 'info_dict': {
830 'id': 'M4gD1WSo5mA',
831 'ext': 'mp4',
832 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
833 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
834 'duration': 721,
835 'upload_date': '20150127',
836 'uploader_id': 'BerkmanCenter',
837 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
838 'uploader': 'The Berkman Klein Center for Internet & Society',
839 'license': 'Creative Commons Attribution license (reuse allowed)',
840 },
841 'params': {
842 'skip_download': True,
843 },
844 },
845 {
846 # Channel-like uploader_url
847 'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
848 'info_dict': {
849 'id': 'eQcmzGIKrzg',
850 'ext': 'mp4',
851 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
852 'description': 'md5:dda0d780d5a6e120758d1711d062a867',
853 'duration': 4060,
854 'upload_date': '20151119',
855 'uploader': 'Bernie 2016',
856 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
857 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
858 'license': 'Creative Commons Attribution license (reuse allowed)',
859 },
860 'params': {
861 'skip_download': True,
862 },
863 },
864 {
865 'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
866 'only_matching': True,
867 },
868 {
869 # YouTube Red paid video (https://github.com/rg3/youtube-dl/issues/10059)
870 'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
871 'only_matching': True,
872 },
873 {
874 # Rental video preview
875 'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
876 'info_dict': {
877 'id': 'uGpuVWrhIzE',
878 'ext': 'mp4',
879 'title': 'Piku - Trailer',
880 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
881 'upload_date': '20150811',
882 'uploader': 'FlixMatrix',
883 'uploader_id': 'FlixMatrixKaravan',
884 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
885 'license': 'Standard YouTube License',
886 },
887 'params': {
888 'skip_download': True,
889 },
890 },
891 {
892 # YouTube Red video with episode data
893 'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
894 'info_dict': {
895 'id': 'iqKdEhx-dD4',
896 'ext': 'mp4',
897 'title': 'Isolation - Mind Field (Ep 1)',
898 'description': 'md5:8013b7ddea787342608f63a13ddc9492',
899 'duration': 2085,
900 'upload_date': '20170118',
901 'uploader': 'Vsauce',
902 'uploader_id': 'Vsauce',
903 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
904 'license': 'Standard YouTube License',
905 'series': 'Mind Field',
906 'season_number': 1,
907 'episode_number': 1,
908 },
909 'params': {
910 'skip_download': True,
911 },
912 'expected_warnings': [
913 'Skipping DASH manifest',
914 ],
915 },
916 {
917 # itag 212
918 'url': '1t24XAntNCY',
919 'only_matching': True,
920 }
921 ]
922
923 def __init__(self, *args, **kwargs):
924 super(YoutubeIE, self).__init__(*args, **kwargs)
925 self._player_cache = {}
926
927 def report_video_info_webpage_download(self, video_id):
928 """Report attempt to download video info webpage."""
929 self.to_screen('%s: Downloading video info webpage' % video_id)
930
931 def report_information_extraction(self, video_id):
932 """Report attempt to extract video information."""
933 self.to_screen('%s: Extracting video information' % video_id)
934
935 def report_unavailable_format(self, video_id, format):
936 """Report extracted video URL."""
937 self.to_screen('%s: Format %s not available' % (video_id, format))
938
939 def report_rtmp_download(self):
940 """Indicate the download will use the RTMP protocol."""
941 self.to_screen('RTMP download detected')
942
943 def _signature_cache_id(self, example_sig):
944 """ Return a string representation of a signature """
945 return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
946
947 def _extract_signature_function(self, video_id, player_url, example_sig):
948 id_m = re.match(
949 r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
950 player_url)
951 if not id_m:
952 raise ExtractorError('Cannot identify player %r' % player_url)
953 player_type = id_m.group('ext')
954 player_id = id_m.group('id')
955
956 # Read from filesystem cache
957 func_id = '%s_%s_%s' % (
958 player_type, player_id, self._signature_cache_id(example_sig))
959 assert os.path.basename(func_id) == func_id
960
961 cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
962 if cache_spec is not None:
963 return lambda s: ''.join(s[i] for i in cache_spec)
964
965 download_note = (
966 'Downloading player %s' % player_url
967 if self._downloader.params.get('verbose') else
968 'Downloading %s player %s' % (player_type, player_id)
969 )
970 if player_type == 'js':
971 code = self._download_webpage(
972 player_url, video_id,
973 note=download_note,
974 errnote='Download of %s failed' % player_url)
975 res = self._parse_sig_js(code)
976 elif player_type == 'swf':
977 urlh = self._request_webpage(
978 player_url, video_id,
979 note=download_note,
980 errnote='Download of %s failed' % player_url)
981 code = urlh.read()
982 res = self._parse_sig_swf(code)
983 else:
984 assert False, 'Invalid player type %r' % player_type
985
986 test_string = ''.join(map(compat_chr, range(len(example_sig))))
987 cache_res = res(test_string)
988 cache_spec = [ord(c) for c in cache_res]
989
990 self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
991 return res
992
993 def _print_sig_code(self, func, example_sig):
994 def gen_sig_code(idxs):
995 def _genslice(start, end, step):
996 starts = '' if start == 0 else str(start)
997 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
998 steps = '' if step == 1 else (':%d' % step)
999 return 's[%s%s%s]' % (starts, ends, steps)
1000
1001 step = None
1002 # Quelch pyflakes warnings - start will be set when step is set
1003 start = '(Never used)'
1004 for i, prev in zip(idxs[1:], idxs[:-1]):
1005 if step is not None:
1006 if i - prev == step:
1007 continue
1008 yield _genslice(start, prev, step)
1009 step = None
1010 continue
1011 if i - prev in [-1, 1]:
1012 step = i - prev
1013 start = prev
1014 continue
1015 else:
1016 yield 's[%d]' % prev
1017 if step is None:
1018 yield 's[%d]' % i
1019 else:
1020 yield _genslice(start, i, step)
1021
1022 test_string = ''.join(map(compat_chr, range(len(example_sig))))
1023 cache_res = func(test_string)
1024 cache_spec = [ord(c) for c in cache_res]
1025 expr_code = ' + '.join(gen_sig_code(cache_spec))
1026 signature_id_tuple = '(%s)' % (
1027 ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1028 code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1029 ' return %s\n') % (signature_id_tuple, expr_code)
1030 self.to_screen('Extracted signature function:\n' + code)
1031
1032 def _parse_sig_js(self, jscode):
1033 funcname = self._search_regex(
1034 (r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1035 r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\('),
1036 jscode, 'Initial JS player signature function name', group='sig')
1037
1038 jsi = JSInterpreter(jscode)
1039 initial_function = jsi.extract_function(funcname)
1040 return lambda s: initial_function([s])
1041
1042 def _parse_sig_swf(self, file_contents):
1043 swfi = SWFInterpreter(file_contents)
1044 TARGET_CLASSNAME = 'SignatureDecipher'
1045 searched_class = swfi.extract_class(TARGET_CLASSNAME)
1046 initial_function = swfi.extract_function(searched_class, 'decipher')
1047 return lambda s: initial_function([s])
1048
1049 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1050 """Turn the encrypted s field into a working signature"""
1051
1052 if player_url is None:
1053 raise ExtractorError('Cannot decrypt signature without player_url')
1054
1055 if player_url.startswith('//'):
1056 player_url = 'https:' + player_url
1057 elif not re.match(r'https?://', player_url):
1058 player_url = compat_urlparse.urljoin(
1059 'https://www.youtube.com', player_url)
1060 try:
1061 player_id = (player_url, self._signature_cache_id(s))
1062 if player_id not in self._player_cache:
1063 func = self._extract_signature_function(
1064 video_id, player_url, s
1065 )
1066 self._player_cache[player_id] = func
1067 func = self._player_cache[player_id]
1068 if self._downloader.params.get('youtube_print_sig_code'):
1069 self._print_sig_code(func, s)
1070 return func(s)
1071 except Exception as e:
1072 tb = traceback.format_exc()
1073 raise ExtractorError(
1074 'Signature extraction failed: ' + tb, cause=e)
1075
1076 def _get_subtitles(self, video_id, webpage):
1077 try:
1078 subs_doc = self._download_xml(
1079 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1080 video_id, note=False)
1081 except ExtractorError as err:
1082 self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
1083 return {}
1084
1085 sub_lang_list = {}
1086 for track in subs_doc.findall('track'):
1087 lang = track.attrib['lang_code']
1088 if lang in sub_lang_list:
1089 continue
1090 sub_formats = []
1091 for ext in self._SUBTITLE_FORMATS:
1092 params = compat_urllib_parse_urlencode({
1093 'lang': lang,
1094 'v': video_id,
1095 'fmt': ext,
1096 'name': track.attrib['name'].encode('utf-8'),
1097 })
1098 sub_formats.append({
1099 'url': 'https://www.youtube.com/api/timedtext?' + params,
1100 'ext': ext,
1101 })
1102 sub_lang_list[lang] = sub_formats
1103 if not sub_lang_list:
1104 self._downloader.report_warning('video doesn\'t have subtitles')
1105 return {}
1106 return sub_lang_list
1107
1108 def _get_ytplayer_config(self, video_id, webpage):
1109 patterns = (
1110 # User data may contain arbitrary character sequences that may affect
1111 # JSON extraction with regex, e.g. when '};' is contained the second
1112 # regex won't capture the whole JSON. Yet working around by trying more
1113 # concrete regex first keeping in mind proper quoted string handling
1114 # to be implemented in future that will replace this workaround (see
1115 # https://github.com/rg3/youtube-dl/issues/7468,
1116 # https://github.com/rg3/youtube-dl/pull/7599)
1117 r';ytplayer\.config\s*=\s*({.+?});ytplayer',
1118 r';ytplayer\.config\s*=\s*({.+?});',
1119 )
1120 config = self._search_regex(
1121 patterns, webpage, 'ytplayer.config', default=None)
1122 if config:
1123 return self._parse_json(
1124 uppercase_escape(config), video_id, fatal=False)
1125
1126 def _get_automatic_captions(self, video_id, webpage):
1127 """We need the webpage for getting the captions url, pass it as an
1128 argument to speed up the process."""
1129 self.to_screen('%s: Looking for automatic captions' % video_id)
1130 player_config = self._get_ytplayer_config(video_id, webpage)
1131 err_msg = 'Couldn\'t find automatic captions for %s' % video_id
1132 if not player_config:
1133 self._downloader.report_warning(err_msg)
1134 return {}
1135 try:
1136 args = player_config['args']
1137 caption_url = args.get('ttsurl')
1138 if caption_url:
1139 timestamp = args['timestamp']
1140 # We get the available subtitles
1141 list_params = compat_urllib_parse_urlencode({
1142 'type': 'list',
1143 'tlangs': 1,
1144 'asrs': 1,
1145 })
1146 list_url = caption_url + '&' + list_params
1147 caption_list = self._download_xml(list_url, video_id)
1148 original_lang_node = caption_list.find('track')
1149 if original_lang_node is None:
1150 self._downloader.report_warning('Video doesn\'t have automatic captions')
1151 return {}
1152 original_lang = original_lang_node.attrib['lang_code']
1153 caption_kind = original_lang_node.attrib.get('kind', '')
1154
1155 sub_lang_list = {}
1156 for lang_node in caption_list.findall('target'):
1157 sub_lang = lang_node.attrib['lang_code']
1158 sub_formats = []
1159 for ext in self._SUBTITLE_FORMATS:
1160 params = compat_urllib_parse_urlencode({
1161 'lang': original_lang,
1162 'tlang': sub_lang,
1163 'fmt': ext,
1164 'ts': timestamp,
1165 'kind': caption_kind,
1166 })
1167 sub_formats.append({
1168 'url': caption_url + '&' + params,
1169 'ext': ext,
1170 })
1171 sub_lang_list[sub_lang] = sub_formats
1172 return sub_lang_list
1173
1174 # Some videos don't provide ttsurl but rather caption_tracks and
1175 # caption_translation_languages (e.g. 20LmZk1hakA)
1176 caption_tracks = args['caption_tracks']
1177 caption_translation_languages = args['caption_translation_languages']
1178 caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
1179 parsed_caption_url = compat_urllib_parse_urlparse(caption_url)
1180 caption_qs = compat_parse_qs(parsed_caption_url.query)
1181
1182 sub_lang_list = {}
1183 for lang in caption_translation_languages.split(','):
1184 lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
1185 sub_lang = lang_qs.get('lc', [None])[0]
1186 if not sub_lang:
1187 continue
1188 sub_formats = []
1189 for ext in self._SUBTITLE_FORMATS:
1190 caption_qs.update({
1191 'tlang': [sub_lang],
1192 'fmt': [ext],
1193 })
1194 sub_url = compat_urlparse.urlunparse(parsed_caption_url._replace(
1195 query=compat_urllib_parse_urlencode(caption_qs, True)))
1196 sub_formats.append({
1197 'url': sub_url,
1198 'ext': ext,
1199 })
1200 sub_lang_list[sub_lang] = sub_formats
1201 return sub_lang_list
1202 # An extractor error can be raise by the download process if there are
1203 # no automatic captions but there are subtitles
1204 except (KeyError, ExtractorError):
1205 self._downloader.report_warning(err_msg)
1206 return {}
1207
1208 def _mark_watched(self, video_id, video_info):
1209 playback_url = video_info.get('videostats_playback_base_url', [None])[0]
1210 if not playback_url:
1211 return
1212 parsed_playback_url = compat_urlparse.urlparse(playback_url)
1213 qs = compat_urlparse.parse_qs(parsed_playback_url.query)
1214
1215 # cpn generation algorithm is reverse engineered from base.js.
1216 # In fact it works even with dummy cpn.
1217 CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1218 cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
1219
1220 qs.update({
1221 'ver': ['2'],
1222 'cpn': [cpn],
1223 })
1224 playback_url = compat_urlparse.urlunparse(
1225 parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
1226
1227 self._download_webpage(
1228 playback_url, video_id, 'Marking watched',
1229 'Unable to mark watched', fatal=False)
1230
1231 @classmethod
1232 def extract_id(cls, url):
1233 mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1234 if mobj is None:
1235 raise ExtractorError('Invalid URL: %s' % url)
1236 video_id = mobj.group(2)
1237 return video_id
1238
1239 def _extract_from_m3u8(self, manifest_url, video_id):
1240 url_map = {}
1241
1242 def _get_urls(_manifest):
1243 lines = _manifest.split('\n')
1244 urls = filter(lambda l: l and not l.startswith('#'),
1245 lines)
1246 return urls
1247 manifest = self._download_webpage(manifest_url, video_id, 'Downloading formats manifest')
1248 formats_urls = _get_urls(manifest)
1249 for format_url in formats_urls:
1250 itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
1251 url_map[itag] = format_url
1252 return url_map
1253
1254 def _extract_annotations(self, video_id):
1255 url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1256 return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
1257
1258 def _real_extract(self, url):
1259 url, smuggled_data = unsmuggle_url(url, {})
1260
1261 proto = (
1262 'http' if self._downloader.params.get('prefer_insecure', False)
1263 else 'https')
1264
1265 start_time = None
1266 end_time = None
1267 parsed_url = compat_urllib_parse_urlparse(url)
1268 for component in [parsed_url.fragment, parsed_url.query]:
1269 query = compat_parse_qs(component)
1270 if start_time is None and 't' in query:
1271 start_time = parse_duration(query['t'][0])
1272 if start_time is None and 'start' in query:
1273 start_time = parse_duration(query['start'][0])
1274 if end_time is None and 'end' in query:
1275 end_time = parse_duration(query['end'][0])
1276
1277 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1278 mobj = re.search(self._NEXT_URL_RE, url)
1279 if mobj:
1280 url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
1281 video_id = self.extract_id(url)
1282
1283 # Get video webpage
1284 url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1285 video_webpage = self._download_webpage(url, video_id)
1286
1287 # Attempt to extract SWF player URL
1288 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1289 if mobj is not None:
1290 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1291 else:
1292 player_url = None
1293
1294 dash_mpds = []
1295
1296 def add_dash_mpd(video_info):
1297 dash_mpd = video_info.get('dashmpd')
1298 if dash_mpd and dash_mpd[0] not in dash_mpds:
1299 dash_mpds.append(dash_mpd[0])
1300
1301 # Get video info
1302 embed_webpage = None
1303 is_live = None
1304 if re.search(r'player-age-gate-content">', video_webpage) is not None:
1305 age_gate = True
1306 # We simulate the access to the video from www.youtube.com/v/{video_id}
1307 # this can be viewed without login into Youtube
1308 url = proto + '://www.youtube.com/embed/%s' % video_id
1309 embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
1310 data = compat_urllib_parse_urlencode({
1311 'video_id': video_id,
1312 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1313 'sts': self._search_regex(
1314 r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
1315 })
1316 video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1317 video_info_webpage = self._download_webpage(
1318 video_info_url, video_id,
1319 note='Refetching age-gated info webpage',
1320 errnote='unable to download video info webpage')
1321 video_info = compat_parse_qs(video_info_webpage)
1322 add_dash_mpd(video_info)
1323 else:
1324 age_gate = False
1325 video_info = None
1326 # Try looking directly into the video webpage
1327 ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
1328 if ytplayer_config:
1329 args = ytplayer_config['args']
1330 if args.get('url_encoded_fmt_stream_map'):
1331 # Convert to the same format returned by compat_parse_qs
1332 video_info = dict((k, [v]) for k, v in args.items())
1333 add_dash_mpd(video_info)
1334 # Rental video is not rented but preview is available (e.g.
1335 # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
1336 # https://github.com/rg3/youtube-dl/issues/10532)
1337 if not video_info and args.get('ypc_vid'):
1338 return self.url_result(
1339 args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
1340 if args.get('livestream') == '1' or args.get('live_playback') == 1:
1341 is_live = True
1342 if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
1343 # We also try looking in get_video_info since it may contain different dashmpd
1344 # URL that points to a DASH manifest with possibly different itag set (some itags
1345 # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
1346 # manifest pointed by get_video_info's dashmpd).
1347 # The general idea is to take a union of itags of both DASH manifests (for example
1348 # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
1349 self.report_video_info_webpage_download(video_id)
1350 for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
1351 video_info_url = (
1352 '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1353 % (proto, video_id, el_type))
1354 video_info_webpage = self._download_webpage(
1355 video_info_url,
1356 video_id, note=False,
1357 errnote='unable to download video info webpage')
1358 get_video_info = compat_parse_qs(video_info_webpage)
1359 if get_video_info.get('use_cipher_signature') != ['True']:
1360 add_dash_mpd(get_video_info)
1361 if not video_info:
1362 video_info = get_video_info
1363 if 'token' in get_video_info:
1364 # Different get_video_info requests may report different results, e.g.
1365 # some may report video unavailability, but some may serve it without
1366 # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
1367 # the original webpage as well as el=info and el=embedded get_video_info
1368 # requests report video unavailability due to geo restriction while
1369 # el=detailpage succeeds and returns valid data). This is probably
1370 # due to YouTube measures against IP ranges of hosting providers.
1371 # Working around by preferring the first succeeded video_info containing
1372 # the token if no such video_info yet was found.
1373 if 'token' not in video_info:
1374 video_info = get_video_info
1375 break
1376 if 'token' not in video_info:
1377 if 'reason' in video_info:
1378 if 'The uploader has not made this video available in your country.' in video_info['reason']:
1379 regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
1380 if regions_allowed:
1381 raise ExtractorError('YouTube said: This video is available in %s only' % (
1382 ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
1383 expected=True)
1384 raise ExtractorError(
1385 'YouTube said: %s' % video_info['reason'][0],
1386 expected=True, video_id=video_id)
1387 else:
1388 raise ExtractorError(
1389 '"token" parameter not in video info for unknown reason',
1390 video_id=video_id)
1391
1392 # title
1393 if 'title' in video_info:
1394 video_title = video_info['title'][0]
1395 else:
1396 self._downloader.report_warning('Unable to extract video title')
1397 video_title = '_'
1398
1399 # description
1400 video_description = get_element_by_id("eow-description", video_webpage)
1401 if video_description:
1402 video_description = re.sub(r'''(?x)
1403 <a\s+
1404 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1405 (?:title|href)="([^"]+)"\s+
1406 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1407 class="[^"]*"[^>]*>
1408 [^<]+\.{3}\s*
1409 </a>
1410 ''', r'\1', video_description)
1411 video_description = clean_html(video_description)
1412 else:
1413 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1414 if fd_mobj:
1415 video_description = unescapeHTML(fd_mobj.group(1))
1416 else:
1417 video_description = ''
1418
1419 if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
1420 if not self._downloader.params.get('noplaylist'):
1421 entries = []
1422 feed_ids = []
1423 multifeed_metadata_list = video_info['multifeed_metadata_list'][0]
1424 for feed in multifeed_metadata_list.split(','):
1425 # Unquote should take place before split on comma (,) since textual
1426 # fields may contain comma as well (see
1427 # https://github.com/rg3/youtube-dl/issues/8536)
1428 feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
1429 entries.append({
1430 '_type': 'url_transparent',
1431 'ie_key': 'Youtube',
1432 'url': smuggle_url(
1433 '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
1434 {'force_singlefeed': True}),
1435 'title': '%s (%s)' % (video_title, feed_data['title'][0]),
1436 })
1437 feed_ids.append(feed_data['id'][0])
1438 self.to_screen(
1439 'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1440 % (', '.join(feed_ids), video_id))
1441 return self.playlist_result(entries, video_id, video_title, video_description)
1442 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1443
1444 if 'view_count' in video_info:
1445 view_count = int(video_info['view_count'][0])
1446 else:
1447 view_count = None
1448
1449 # Check for "rental" videos
1450 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1451 raise ExtractorError('"rental" videos not supported')
1452
1453 # Start extracting information
1454 self.report_information_extraction(video_id)
1455
1456 # uploader
1457 if 'author' not in video_info:
1458 raise ExtractorError('Unable to extract uploader name')
1459 video_uploader = compat_urllib_parse_unquote_plus(video_info['author'][0])
1460
1461 # uploader_id
1462 video_uploader_id = None
1463 video_uploader_url = None
1464 mobj = re.search(
1465 r'<link itemprop="url" href="(?P<uploader_url>https?://www.youtube.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
1466 video_webpage)
1467 if mobj is not None:
1468 video_uploader_id = mobj.group('uploader_id')
1469 video_uploader_url = mobj.group('uploader_url')
1470 else:
1471 self._downloader.report_warning('unable to extract uploader nickname')
1472
1473 # thumbnail image
1474 # We try first to get a high quality image:
1475 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1476 video_webpage, re.DOTALL)
1477 if m_thumb is not None:
1478 video_thumbnail = m_thumb.group(1)
1479 elif 'thumbnail_url' not in video_info:
1480 self._downloader.report_warning('unable to extract video thumbnail')
1481 video_thumbnail = None
1482 else: # don't panic if we can't find it
1483 video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
1484
1485 # upload date
1486 upload_date = self._html_search_meta(
1487 'datePublished', video_webpage, 'upload date', default=None)
1488 if not upload_date:
1489 upload_date = self._search_regex(
1490 [r'(?s)id="eow-date.*?>(.*?)</span>',
1491 r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
1492 video_webpage, 'upload date', default=None)
1493 if upload_date:
1494 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1495 upload_date = unified_strdate(upload_date)
1496
1497 video_license = self._html_search_regex(
1498 r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
1499 video_webpage, 'license', default=None)
1500
1501 m_music = re.search(
1502 r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
1503 video_webpage)
1504 if m_music:
1505 video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
1506 video_creator = clean_html(m_music.group('creator'))
1507 else:
1508 video_alt_title = video_creator = None
1509
1510 m_episode = re.search(
1511 r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*ā€¢\s*E(?P<episode>\d+)</span>',
1512 video_webpage)
1513 if m_episode:
1514 series = m_episode.group('series')
1515 season_number = int(m_episode.group('season'))
1516 episode_number = int(m_episode.group('episode'))
1517 else:
1518 series = season_number = episode_number = None
1519
1520 m_cat_container = self._search_regex(
1521 r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
1522 video_webpage, 'categories', default=None)
1523 if m_cat_container:
1524 category = self._html_search_regex(
1525 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
1526 default=None)
1527 video_categories = None if category is None else [category]
1528 else:
1529 video_categories = None
1530
1531 video_tags = [
1532 unescapeHTML(m.group('content'))
1533 for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
1534
1535 def _extract_count(count_name):
1536 return str_to_int(self._search_regex(
1537 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
1538 % re.escape(count_name),
1539 video_webpage, count_name, default=None))
1540
1541 like_count = _extract_count('like')
1542 dislike_count = _extract_count('dislike')
1543
1544 # subtitles
1545 video_subtitles = self.extract_subtitles(video_id, video_webpage)
1546 automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
1547
1548 video_duration = try_get(
1549 video_info, lambda x: int_or_none(x['length_seconds'][0]))
1550 if not video_duration:
1551 video_duration = parse_duration(self._html_search_meta(
1552 'duration', video_webpage, 'video duration'))
1553
1554 # annotations
1555 video_annotations = None
1556 if self._downloader.params.get('writeannotations', False):
1557 video_annotations = self._extract_annotations(video_id)
1558
1559 def _map_to_format_list(urlmap):
1560 formats = []
1561 for itag, video_real_url in urlmap.items():
1562 dct = {
1563 'format_id': itag,
1564 'url': video_real_url,
1565 'player_url': player_url,
1566 }
1567 if itag in self._formats:
1568 dct.update(self._formats[itag])
1569 formats.append(dct)
1570 return formats
1571
1572 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1573 self.report_rtmp_download()
1574 formats = [{
1575 'format_id': '_rtmp',
1576 'protocol': 'rtmp',
1577 'url': video_info['conn'][0],
1578 'player_url': player_url,
1579 }]
1580 elif len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1:
1581 encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
1582 if 'rtmpe%3Dyes' in encoded_url_map:
1583 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1584 formats_spec = {}
1585 fmt_list = video_info.get('fmt_list', [''])[0]
1586 if fmt_list:
1587 for fmt in fmt_list.split(','):
1588 spec = fmt.split('/')
1589 if len(spec) > 1:
1590 width_height = spec[1].split('x')
1591 if len(width_height) == 2:
1592 formats_spec[spec[0]] = {
1593 'resolution': spec[1],
1594 'width': int_or_none(width_height[0]),
1595 'height': int_or_none(width_height[1]),
1596 }
1597 formats = []
1598 for url_data_str in encoded_url_map.split(','):
1599 url_data = compat_parse_qs(url_data_str)
1600 if 'itag' not in url_data or 'url' not in url_data:
1601 continue
1602 format_id = url_data['itag'][0]
1603 url = url_data['url'][0]
1604
1605 if 'sig' in url_data:
1606 url += '&signature=' + url_data['sig'][0]
1607 elif 's' in url_data:
1608 encrypted_sig = url_data['s'][0]
1609 ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
1610
1611 jsplayer_url_json = self._search_regex(
1612 ASSETS_RE,
1613 embed_webpage if age_gate else video_webpage,
1614 'JS player URL (1)', default=None)
1615 if not jsplayer_url_json and not age_gate:
1616 # We need the embed website after all
1617 if embed_webpage is None:
1618 embed_url = proto + '://www.youtube.com/embed/%s' % video_id
1619 embed_webpage = self._download_webpage(
1620 embed_url, video_id, 'Downloading embed webpage')
1621 jsplayer_url_json = self._search_regex(
1622 ASSETS_RE, embed_webpage, 'JS player URL')
1623
1624 player_url = json.loads(jsplayer_url_json)
1625 if player_url is None:
1626 player_url_json = self._search_regex(
1627 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
1628 video_webpage, 'age gate player URL')
1629 player_url = json.loads(player_url_json)
1630
1631 if self._downloader.params.get('verbose'):
1632 if player_url is None:
1633 player_version = 'unknown'
1634 player_desc = 'unknown'
1635 else:
1636 if player_url.endswith('swf'):
1637 player_version = self._search_regex(
1638 r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
1639 'flash player', fatal=False)
1640 player_desc = 'flash player %s' % player_version
1641 else:
1642 player_version = self._search_regex(
1643 [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
1644 player_url,
1645 'html5 player', fatal=False)
1646 player_desc = 'html5 player %s' % player_version
1647
1648 parts_sizes = self._signature_cache_id(encrypted_sig)
1649 self.to_screen('{%s} signature length %s, %s' %
1650 (format_id, parts_sizes, player_desc))
1651
1652 signature = self._decrypt_signature(
1653 encrypted_sig, video_id, player_url, age_gate)
1654 url += '&signature=' + signature
1655 if 'ratebypass' not in url:
1656 url += '&ratebypass=yes'
1657
1658 dct = {
1659 'format_id': format_id,
1660 'url': url,
1661 'player_url': player_url,
1662 }
1663 if format_id in self._formats:
1664 dct.update(self._formats[format_id])
1665 if format_id in formats_spec:
1666 dct.update(formats_spec[format_id])
1667
1668 # Some itags are not included in DASH manifest thus corresponding formats will
1669 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
1670 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
1671 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
1672 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
1673
1674 more_fields = {
1675 'filesize': int_or_none(url_data.get('clen', [None])[0]),
1676 'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
1677 'width': width,
1678 'height': height,
1679 'fps': int_or_none(url_data.get('fps', [None])[0]),
1680 'format_note': url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0],
1681 }
1682 for key, value in more_fields.items():
1683 if value:
1684 dct[key] = value
1685 type_ = url_data.get('type', [None])[0]
1686 if type_:
1687 type_split = type_.split(';')
1688 kind_ext = type_split[0].split('/')
1689 if len(kind_ext) == 2:
1690 kind, _ = kind_ext
1691 dct['ext'] = mimetype2ext(type_split[0])
1692 if kind in ('audio', 'video'):
1693 codecs = None
1694 for mobj in re.finditer(
1695 r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
1696 if mobj.group('key') == 'codecs':
1697 codecs = mobj.group('val')
1698 break
1699 if codecs:
1700 dct.update(parse_codecs(codecs))
1701 formats.append(dct)
1702 elif video_info.get('hlsvp'):
1703 manifest_url = video_info['hlsvp'][0]
1704 url_map = self._extract_from_m3u8(manifest_url, video_id)
1705 formats = _map_to_format_list(url_map)
1706 # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
1707 for a_format in formats:
1708 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
1709 else:
1710 unavailable_message = self._html_search_regex(
1711 r'(?s)<h1[^>]+id="unavailable-message"[^>]*>(.+?)</h1>',
1712 video_webpage, 'unavailable message', default=None)
1713 if unavailable_message:
1714 raise ExtractorError(unavailable_message, expected=True)
1715 raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1716
1717 # Look for the DASH manifest
1718 if self._downloader.params.get('youtube_include_dash_manifest', True):
1719 dash_mpd_fatal = True
1720 for mpd_url in dash_mpds:
1721 dash_formats = {}
1722 try:
1723 def decrypt_sig(mobj):
1724 s = mobj.group(1)
1725 dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
1726 return '/signature/%s' % dec_s
1727
1728 mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
1729
1730 for df in self._extract_mpd_formats(
1731 mpd_url, video_id, fatal=dash_mpd_fatal,
1732 formats_dict=self._formats):
1733 # Do not overwrite DASH format found in some previous DASH manifest
1734 if df['format_id'] not in dash_formats:
1735 dash_formats[df['format_id']] = df
1736 # Additional DASH manifests may end up in HTTP Error 403 therefore
1737 # allow them to fail without bug report message if we already have
1738 # some DASH manifest succeeded. This is temporary workaround to reduce
1739 # burst of bug reports until we figure out the reason and whether it
1740 # can be fixed at all.
1741 dash_mpd_fatal = False
1742 except (ExtractorError, KeyError) as e:
1743 self.report_warning(
1744 'Skipping DASH manifest: %r' % e, video_id)
1745 if dash_formats:
1746 # Remove the formats we found through non-DASH, they
1747 # contain less info and it can be wrong, because we use
1748 # fixed values (for example the resolution). See
1749 # https://github.com/rg3/youtube-dl/issues/5774 for an
1750 # example.
1751 formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
1752 formats.extend(dash_formats.values())
1753
1754 # Check for malformed aspect ratio
1755 stretched_m = re.search(
1756 r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
1757 video_webpage)
1758 if stretched_m:
1759 w = float(stretched_m.group('w'))
1760 h = float(stretched_m.group('h'))
1761 # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
1762 # We will only process correct ratios.
1763 if w > 0 and h > 0:
1764 ratio = w / h
1765 for f in formats:
1766 if f.get('vcodec') != 'none':
1767 f['stretched_ratio'] = ratio
1768
1769 self._sort_formats(formats)
1770
1771 self.mark_watched(video_id, video_info)
1772
1773 return {
1774 'id': video_id,
1775 'uploader': video_uploader,
1776 'uploader_id': video_uploader_id,
1777 'uploader_url': video_uploader_url,
1778 'upload_date': upload_date,
1779 'license': video_license,
1780 'creator': video_creator,
1781 'title': video_title,
1782 'alt_title': video_alt_title,
1783 'thumbnail': video_thumbnail,
1784 'description': video_description,
1785 'categories': video_categories,
1786 'tags': video_tags,
1787 'subtitles': video_subtitles,
1788 'automatic_captions': automatic_captions,
1789 'duration': video_duration,
1790 'age_limit': 18 if age_gate else 0,
1791 'annotations': video_annotations,
1792 'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
1793 'view_count': view_count,
1794 'like_count': like_count,
1795 'dislike_count': dislike_count,
1796 'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
1797 'formats': formats,
1798 'is_live': is_live,
1799 'start_time': start_time,
1800 'end_time': end_time,
1801 'series': series,
1802 'season_number': season_number,
1803 'episode_number': episode_number,
1804 }
1805
1806
1807 class YoutubeSharedVideoIE(InfoExtractor):
1808 _VALID_URL = r'(?:https?:)?//(?:www\.)?youtube\.com/shared\?.*\bci=(?P<id>[0-9A-Za-z_-]{11})'
1809 IE_NAME = 'youtube:shared'
1810
1811 _TEST = {
1812 'url': 'https://www.youtube.com/shared?ci=1nEzmT-M4fU',
1813 'info_dict': {
1814 'id': 'uPDB5I9wfp8',
1815 'ext': 'webm',
1816 'title': 'Pocoyo: 90 minutos de episĆ³dios completos PortuguĆŖs para crianƧas - PARTE 3',
1817 'description': 'md5:d9e4d9346a2dfff4c7dc4c8cec0f546d',
1818 'upload_date': '20160219',
1819 'uploader': 'Pocoyo - PortuguĆŖs (BR)',
1820 'uploader_id': 'PocoyoBrazil',
1821 },
1822 'add_ie': ['Youtube'],
1823 'params': {
1824 # There are already too many Youtube downloads
1825 'skip_download': True,
1826 },
1827 }
1828
1829 def _real_extract(self, url):
1830 video_id = self._match_id(url)
1831
1832 webpage = self._download_webpage(url, video_id)
1833
1834 real_video_id = self._html_search_meta(
1835 'videoId', webpage, 'YouTube video id', fatal=True)
1836
1837 return self.url_result(real_video_id, YoutubeIE.ie_key())
1838
1839
1840 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
1841 IE_DESC = 'YouTube.com playlists'
1842 _VALID_URL = r"""(?x)(?:
1843 (?:https?://)?
1844 (?:\w+\.)?
1845 (?:
1846 youtube\.com/
1847 (?:
1848 (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
1849 \? (?:.*?[&;])*? (?:p|a|list)=
1850 | p/
1851 )|
1852 youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
1853 )
1854 (
1855 (?:PL|LL|EC|UU|FL|RD|UL|TL)?[0-9A-Za-z-_]{10,}
1856 # Top tracks, they can also include dots
1857 |(?:MC)[\w\.]*
1858 )
1859 .*
1860 |
1861 ((?:PL|LL|EC|UU|FL|RD|UL|TL)[0-9A-Za-z-_]{10,})
1862 )"""
1863 _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&disable_polymer=true'
1864 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
1865 IE_NAME = 'youtube:playlist'
1866 _TESTS = [{
1867 'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1868 'info_dict': {
1869 'title': 'ytdl test PL',
1870 'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1871 },
1872 'playlist_count': 3,
1873 }, {
1874 'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
1875 'info_dict': {
1876 'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
1877 'title': 'YDL_Empty_List',
1878 },
1879 'playlist_count': 0,
1880 'skip': 'This playlist is private',
1881 }, {
1882 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
1883 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1884 'info_dict': {
1885 'title': '29C3: Not my department',
1886 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1887 },
1888 'playlist_count': 95,
1889 }, {
1890 'note': 'issue #673',
1891 'url': 'PLBB231211A4F62143',
1892 'info_dict': {
1893 'title': '[OLD]Team Fortress 2 (Class-based LP)',
1894 'id': 'PLBB231211A4F62143',
1895 },
1896 'playlist_mincount': 26,
1897 }, {
1898 'note': 'Large playlist',
1899 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
1900 'info_dict': {
1901 'title': 'Uploads from Cauchemar',
1902 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
1903 },
1904 'playlist_mincount': 799,
1905 }, {
1906 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1907 'info_dict': {
1908 'title': 'YDL_safe_search',
1909 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1910 },
1911 'playlist_count': 2,
1912 'skip': 'This playlist is private',
1913 }, {
1914 'note': 'embedded',
1915 'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1916 'playlist_count': 4,
1917 'info_dict': {
1918 'title': 'JODA15',
1919 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1920 }
1921 }, {
1922 'note': 'Embedded SWF player',
1923 'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
1924 'playlist_count': 4,
1925 'info_dict': {
1926 'title': 'JODA7',
1927 'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
1928 }
1929 }, {
1930 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
1931 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
1932 'info_dict': {
1933 'title': 'Uploads from Interstellar Movie',
1934 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
1935 },
1936 'playlist_mincount': 21,
1937 }, {
1938 # Playlist URL that does not actually serve a playlist
1939 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
1940 'info_dict': {
1941 'id': 'FqZTN594JQw',
1942 'ext': 'webm',
1943 'title': "Smiley's People 01 detective, Adventure Series, Action",
1944 'uploader': 'STREEM',
1945 'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
1946 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
1947 'upload_date': '20150526',
1948 'license': 'Standard YouTube License',
1949 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
1950 'categories': ['People & Blogs'],
1951 'tags': list,
1952 'like_count': int,
1953 'dislike_count': int,
1954 },
1955 'params': {
1956 'skip_download': True,
1957 },
1958 'add_ie': [YoutubeIE.ie_key()],
1959 }, {
1960 'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
1961 'info_dict': {
1962 'id': 'yeWKywCrFtk',
1963 'ext': 'mp4',
1964 'title': 'Small Scale Baler and Braiding Rugs',
1965 'uploader': 'Backus-Page House Museum',
1966 'uploader_id': 'backuspagemuseum',
1967 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
1968 'upload_date': '20161008',
1969 'license': 'Standard YouTube License',
1970 'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
1971 'categories': ['Nonprofits & Activism'],
1972 'tags': list,
1973 'like_count': int,
1974 'dislike_count': int,
1975 },
1976 'params': {
1977 'noplaylist': True,
1978 'skip_download': True,
1979 },
1980 }, {
1981 'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
1982 'only_matching': True,
1983 }, {
1984 'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
1985 'only_matching': True,
1986 }]
1987
1988 def _real_initialize(self):
1989 self._login()
1990
1991 def _extract_mix(self, playlist_id):
1992 # The mixes are generated from a single video
1993 # the id of the playlist is just 'RD' + video_id
1994 ids = []
1995 last_id = playlist_id[-11:]
1996 for n in itertools.count(1):
1997 url = 'https://youtube.com/watch?v=%s&list=%s' % (last_id, playlist_id)
1998 webpage = self._download_webpage(
1999 url, playlist_id, 'Downloading page {0} of Youtube mix'.format(n))
2000 new_ids = orderedSet(re.findall(
2001 r'''(?xs)data-video-username=".*?".*?
2002 href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
2003 webpage))
2004 # Fetch new pages until all the videos are repeated, it seems that
2005 # there are always 51 unique videos.
2006 new_ids = [_id for _id in new_ids if _id not in ids]
2007 if not new_ids:
2008 break
2009 ids.extend(new_ids)
2010 last_id = ids[-1]
2011
2012 url_results = self._ids_to_results(ids)
2013
2014 search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
2015 title_span = (
2016 search_title('playlist-title') or
2017 search_title('title long-title') or
2018 search_title('title'))
2019 title = clean_html(title_span)
2020
2021 return self.playlist_result(url_results, playlist_id, title)
2022
2023 def _extract_playlist(self, playlist_id):
2024 url = self._TEMPLATE_URL % playlist_id
2025 page = self._download_webpage(url, playlist_id)
2026
2027 # the yt-alert-message now has tabindex attribute (see https://github.com/rg3/youtube-dl/issues/11604)
2028 for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
2029 match = match.strip()
2030 # Check if the playlist exists or is private
2031 mobj = re.match(r'[^<]*(?:The|This) playlist (?P<reason>does not exist|is private)[^<]*', match)
2032 if mobj:
2033 reason = mobj.group('reason')
2034 message = 'This playlist %s' % reason
2035 if 'private' in reason:
2036 message += ', use --username or --netrc to access it'
2037 message += '.'
2038 raise ExtractorError(message, expected=True)
2039 elif re.match(r'[^<]*Invalid parameters[^<]*', match):
2040 raise ExtractorError(
2041 'Invalid parameters. Maybe URL is incorrect.',
2042 expected=True)
2043 elif re.match(r'[^<]*Choose your language[^<]*', match):
2044 continue
2045 else:
2046 self.report_warning('Youtube gives an alert message: ' + match)
2047
2048 playlist_title = self._html_search_regex(
2049 r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
2050 page, 'title', default=None)
2051
2052 has_videos = True
2053
2054 if not playlist_title:
2055 try:
2056 # Some playlist URLs don't actually serve a playlist (e.g.
2057 # https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4)
2058 next(self._entries(page, playlist_id))
2059 except StopIteration:
2060 has_videos = False
2061
2062 return has_videos, self.playlist_result(
2063 self._entries(page, playlist_id), playlist_id, playlist_title)
2064
2065 def _check_download_just_video(self, url, playlist_id):
2066 # Check if it's a video-specific URL
2067 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
2068 video_id = query_dict.get('v', [None])[0] or self._search_regex(
2069 r'(?:^|//)youtu\.be/([0-9A-Za-z_-]{11})', url,
2070 'video id', default=None)
2071 if video_id:
2072 if self._downloader.params.get('noplaylist'):
2073 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2074 return video_id, self.url_result(video_id, 'Youtube', video_id=video_id)
2075 else:
2076 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
2077 return video_id, None
2078 return None, None
2079
2080 def _real_extract(self, url):
2081 # Extract playlist id
2082 mobj = re.match(self._VALID_URL, url)
2083 if mobj is None:
2084 raise ExtractorError('Invalid URL: %s' % url)
2085 playlist_id = mobj.group(1) or mobj.group(2)
2086
2087 video_id, video = self._check_download_just_video(url, playlist_id)
2088 if video:
2089 return video
2090
2091 if playlist_id.startswith(('RD', 'UL', 'PU')):
2092 # Mixes require a custom extraction process
2093 return self._extract_mix(playlist_id)
2094
2095 has_videos, playlist = self._extract_playlist(playlist_id)
2096 if has_videos or not video_id:
2097 return playlist
2098
2099 # Some playlist URLs don't actually serve a playlist (see
2100 # https://github.com/rg3/youtube-dl/issues/10537).
2101 # Fallback to plain video extraction if there is a video id
2102 # along with playlist id.
2103 return self.url_result(video_id, 'Youtube', video_id=video_id)
2104
2105
2106 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
2107 IE_DESC = 'YouTube.com channels'
2108 _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
2109 _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
2110 _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
2111 IE_NAME = 'youtube:channel'
2112 _TESTS = [{
2113 'note': 'paginated channel',
2114 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
2115 'playlist_mincount': 91,
2116 'info_dict': {
2117 'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
2118 'title': 'Uploads from lex will',
2119 }
2120 }, {
2121 'note': 'Age restricted channel',
2122 # from https://www.youtube.com/user/DeusExOfficial
2123 'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
2124 'playlist_mincount': 64,
2125 'info_dict': {
2126 'id': 'UUs0ifCMCm1icqRbqhUINa0w',
2127 'title': 'Uploads from Deus Ex',
2128 },
2129 }]
2130
2131 @classmethod
2132 def suitable(cls, url):
2133 return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
2134 else super(YoutubeChannelIE, cls).suitable(url))
2135
2136 def _build_template_url(self, url, channel_id):
2137 return self._TEMPLATE_URL % channel_id
2138
2139 def _real_extract(self, url):
2140 channel_id = self._match_id(url)
2141
2142 url = self._build_template_url(url, channel_id)
2143
2144 # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
2145 # Workaround by extracting as a playlist if managed to obtain channel playlist URL
2146 # otherwise fallback on channel by page extraction
2147 channel_page = self._download_webpage(
2148 url + '?view=57', channel_id,
2149 'Downloading channel page', fatal=False)
2150 if channel_page is False:
2151 channel_playlist_id = False
2152 else:
2153 channel_playlist_id = self._html_search_meta(
2154 'channelId', channel_page, 'channel id', default=None)
2155 if not channel_playlist_id:
2156 channel_url = self._html_search_meta(
2157 ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
2158 channel_page, 'channel url', default=None)
2159 if channel_url:
2160 channel_playlist_id = self._search_regex(
2161 r'vnd\.youtube://user/([0-9A-Za-z_-]+)',
2162 channel_url, 'channel id', default=None)
2163 if channel_playlist_id and channel_playlist_id.startswith('UC'):
2164 playlist_id = 'UU' + channel_playlist_id[2:]
2165 return self.url_result(
2166 compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
2167
2168 channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
2169 autogenerated = re.search(r'''(?x)
2170 class="[^"]*?(?:
2171 channel-header-autogenerated-label|
2172 yt-channel-title-autogenerated
2173 )[^"]*"''', channel_page) is not None
2174
2175 if autogenerated:
2176 # The videos are contained in a single page
2177 # the ajax pages can't be used, they are empty
2178 entries = [
2179 self.url_result(
2180 video_id, 'Youtube', video_id=video_id,
2181 video_title=video_title)
2182 for video_id, video_title in self.extract_videos_from_page(channel_page)]
2183 return self.playlist_result(entries, channel_id)
2184
2185 try:
2186 next(self._entries(channel_page, channel_id))
2187 except StopIteration:
2188 alert_message = self._html_search_regex(
2189 r'(?s)<div[^>]+class=(["\']).*?\byt-alert-message\b.*?\1[^>]*>(?P<alert>[^<]+)</div>',
2190 channel_page, 'alert', default=None, group='alert')
2191 if alert_message:
2192 raise ExtractorError('Youtube said: %s' % alert_message, expected=True)
2193
2194 return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
2195
2196
2197 class YoutubeUserIE(YoutubeChannelIE):
2198 IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
2199 _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:(?P<user>user|c)/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
2200 _TEMPLATE_URL = 'https://www.youtube.com/%s/%s/videos'
2201 IE_NAME = 'youtube:user'
2202
2203 _TESTS = [{
2204 'url': 'https://www.youtube.com/user/TheLinuxFoundation',
2205 'playlist_mincount': 320,
2206 'info_dict': {
2207 'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
2208 'title': 'Uploads from The Linux Foundation',
2209 }
2210 }, {
2211 # Only available via https://www.youtube.com/c/12minuteathlete/videos
2212 # but not https://www.youtube.com/user/12minuteathlete/videos
2213 'url': 'https://www.youtube.com/c/12minuteathlete/videos',
2214 'playlist_mincount': 249,
2215 'info_dict': {
2216 'id': 'UUVjM-zV6_opMDx7WYxnjZiQ',
2217 'title': 'Uploads from 12 Minute Athlete',
2218 }
2219 }, {
2220 'url': 'ytuser:phihag',
2221 'only_matching': True,
2222 }, {
2223 'url': 'https://www.youtube.com/c/gametrailers',
2224 'only_matching': True,
2225 }, {
2226 'url': 'https://www.youtube.com/gametrailers',
2227 'only_matching': True,
2228 }, {
2229 # This channel is not available.
2230 'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
2231 'only_matching': True,
2232 }]
2233
2234 @classmethod
2235 def suitable(cls, url):
2236 # Don't return True if the url can be extracted with other youtube
2237 # extractor, the regex would is too permissive and it would match.
2238 other_yt_ies = iter(klass for (name, klass) in globals().items() if name.startswith('Youtube') and name.endswith('IE') and klass is not cls)
2239 if any(ie.suitable(url) for ie in other_yt_ies):
2240 return False
2241 else:
2242 return super(YoutubeUserIE, cls).suitable(url)
2243
2244 def _build_template_url(self, url, channel_id):
2245 mobj = re.match(self._VALID_URL, url)
2246 return self._TEMPLATE_URL % (mobj.group('user') or 'user', mobj.group('id'))
2247
2248
2249 class YoutubeLiveIE(YoutubeBaseInfoExtractor):
2250 IE_DESC = 'YouTube.com live streams'
2251 _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:(?:user|channel|c)/)?(?P<id>[^/]+))/live'
2252 IE_NAME = 'youtube:live'
2253
2254 _TESTS = [{
2255 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
2256 'info_dict': {
2257 'id': 'a48o2S1cPoo',
2258 'ext': 'mp4',
2259 'title': 'The Young Turks - Live Main Show',
2260 'uploader': 'The Young Turks',
2261 'uploader_id': 'TheYoungTurks',
2262 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
2263 'upload_date': '20150715',
2264 'license': 'Standard YouTube License',
2265 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
2266 'categories': ['News & Politics'],
2267 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
2268 'like_count': int,
2269 'dislike_count': int,
2270 },
2271 'params': {
2272 'skip_download': True,
2273 },
2274 }, {
2275 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
2276 'only_matching': True,
2277 }, {
2278 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
2279 'only_matching': True,
2280 }, {
2281 'url': 'https://www.youtube.com/TheYoungTurks/live',
2282 'only_matching': True,
2283 }]
2284
2285 def _real_extract(self, url):
2286 mobj = re.match(self._VALID_URL, url)
2287 channel_id = mobj.group('id')
2288 base_url = mobj.group('base_url')
2289 webpage = self._download_webpage(url, channel_id, fatal=False)
2290 if webpage:
2291 page_type = self._og_search_property(
2292 'type', webpage, 'page type', default=None)
2293 video_id = self._html_search_meta(
2294 'videoId', webpage, 'video id', default=None)
2295 if page_type == 'video' and video_id and re.match(r'^[0-9A-Za-z_-]{11}$', video_id):
2296 return self.url_result(video_id, YoutubeIE.ie_key())
2297 return self.url_result(base_url)
2298
2299
2300 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
2301 IE_DESC = 'YouTube.com user/channel playlists'
2302 _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
2303 IE_NAME = 'youtube:playlists'
2304
2305 _TESTS = [{
2306 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
2307 'playlist_mincount': 4,
2308 'info_dict': {
2309 'id': 'ThirstForScience',
2310 'title': 'Thirst for Science',
2311 },
2312 }, {
2313 # with "Load more" button
2314 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
2315 'playlist_mincount': 70,
2316 'info_dict': {
2317 'id': 'igorkle1',
2318 'title': 'Š˜Š³Š¾Ń€ŃŒ ŠšŠ»ŠµŠ¹Š½ŠµŃ€',
2319 },
2320 }, {
2321 'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
2322 'playlist_mincount': 17,
2323 'info_dict': {
2324 'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
2325 'title': 'Chem Player',
2326 },
2327 }]
2328
2329
2330 class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
2331 IE_DESC = 'YouTube.com searches'
2332 # there doesn't appear to be a real limit, for example if you search for
2333 # 'python' you get more than 8.000.000 results
2334 _MAX_RESULTS = float('inf')
2335 IE_NAME = 'youtube:search'
2336 _SEARCH_KEY = 'ytsearch'
2337 _EXTRA_QUERY_ARGS = {}
2338 _TESTS = []
2339
2340 def _get_n_results(self, query, n):
2341 """Get a specified number of results for a query"""
2342
2343 videos = []
2344 limit = n
2345
2346 url_query = {
2347 'search_query': query.encode('utf-8'),
2348 }
2349 url_query.update(self._EXTRA_QUERY_ARGS)
2350 result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
2351
2352 for pagenum in itertools.count(1):
2353 data = self._download_json(
2354 result_url, video_id='query "%s"' % query,
2355 note='Downloading page %s' % pagenum,
2356 errnote='Unable to download API page',
2357 query={'spf': 'navigate'})
2358 html_content = data[1]['body']['content']
2359
2360 if 'class="search-message' in html_content:
2361 raise ExtractorError(
2362 '[youtube] No video results', expected=True)
2363
2364 new_videos = self._ids_to_results(orderedSet(re.findall(
2365 r'href="/watch\?v=(.{11})', html_content)))
2366 videos += new_videos
2367 if not new_videos or len(videos) > limit:
2368 break
2369 next_link = self._html_search_regex(
2370 r'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*<span[^>]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next',
2371 html_content, 'next link', default=None)
2372 if next_link is None:
2373 break
2374 result_url = compat_urlparse.urljoin('https://www.youtube.com/', next_link)
2375
2376 if len(videos) > n:
2377 videos = videos[:n]
2378 return self.playlist_result(videos, query)
2379
2380
2381 class YoutubeSearchDateIE(YoutubeSearchIE):
2382 IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
2383 _SEARCH_KEY = 'ytsearchdate'
2384 IE_DESC = 'YouTube.com searches, newest videos first'
2385 _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
2386
2387
2388 class YoutubeSearchURLIE(YoutubePlaylistBaseInfoExtractor):
2389 IE_DESC = 'YouTube.com search URLs'
2390 IE_NAME = 'youtube:search_url'
2391 _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
2392 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
2393 _TESTS = [{
2394 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
2395 'playlist_mincount': 5,
2396 'info_dict': {
2397 'title': 'youtube-dl test video',
2398 }
2399 }, {
2400 'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
2401 'only_matching': True,
2402 }]
2403
2404 def _real_extract(self, url):
2405 mobj = re.match(self._VALID_URL, url)
2406 query = compat_urllib_parse_unquote_plus(mobj.group('query'))
2407 webpage = self._download_webpage(url, query)
2408 return self.playlist_result(self._process_page(webpage), playlist_title=query)
2409
2410
2411 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
2412 IE_DESC = 'YouTube.com (multi-season) shows'
2413 _VALID_URL = r'https?://(?:www\.)?youtube\.com/show/(?P<id>[^?#]*)'
2414 IE_NAME = 'youtube:show'
2415 _TESTS = [{
2416 'url': 'https://www.youtube.com/show/airdisasters',
2417 'playlist_mincount': 5,
2418 'info_dict': {
2419 'id': 'airdisasters',
2420 'title': 'Air Disasters',
2421 }
2422 }]
2423
2424 def _real_extract(self, url):
2425 playlist_id = self._match_id(url)
2426 return super(YoutubeShowIE, self)._real_extract(
2427 'https://www.youtube.com/show/%s/playlists' % playlist_id)
2428
2429
2430 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
2431 """
2432 Base class for feed extractors
2433 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
2434 """
2435 _LOGIN_REQUIRED = True
2436
2437 @property
2438 def IE_NAME(self):
2439 return 'youtube:%s' % self._FEED_NAME
2440
2441 def _real_initialize(self):
2442 self._login()
2443
2444 def _real_extract(self, url):
2445 page = self._download_webpage(
2446 'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE)
2447
2448 # The extraction process is the same as for playlists, but the regex
2449 # for the video ids doesn't contain an index
2450 ids = []
2451 more_widget_html = content_html = page
2452 for page_num in itertools.count(1):
2453 matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
2454
2455 # 'recommended' feed has infinite 'load more' and each new portion spins
2456 # the same videos in (sometimes) slightly different order, so we'll check
2457 # for unicity and break when portion has no new videos
2458 new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches))
2459 if not new_ids:
2460 break
2461
2462 ids.extend(new_ids)
2463
2464 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
2465 if not mobj:
2466 break
2467
2468 more = self._download_json(
2469 'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
2470 'Downloading page #%s' % page_num,
2471 transform_source=uppercase_escape)
2472 content_html = more['content_html']
2473 more_widget_html = more['load_more_widget_html']
2474
2475 return self.playlist_result(
2476 self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE)
2477
2478
2479 class YoutubeWatchLaterIE(YoutubePlaylistIE):
2480 IE_NAME = 'youtube:watchlater'
2481 IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
2482 _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
2483
2484 _TESTS = [{
2485 'url': 'https://www.youtube.com/playlist?list=WL',
2486 'only_matching': True,
2487 }, {
2488 'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
2489 'only_matching': True,
2490 }]
2491
2492 def _real_extract(self, url):
2493 _, video = self._check_download_just_video(url, 'WL')
2494 if video:
2495 return video
2496 _, playlist = self._extract_playlist('WL')
2497 return playlist
2498
2499
2500 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
2501 IE_NAME = 'youtube:favorites'
2502 IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
2503 _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
2504 _LOGIN_REQUIRED = True
2505
2506 def _real_extract(self, url):
2507 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
2508 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
2509 return self.url_result(playlist_id, 'YoutubePlaylist')
2510
2511
2512 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
2513 IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
2514 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/recommended|:ytrec(?:ommended)?'
2515 _FEED_NAME = 'recommended'
2516 _PLAYLIST_TITLE = 'Youtube Recommended videos'
2517
2518
2519 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
2520 IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
2521 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
2522 _FEED_NAME = 'subscriptions'
2523 _PLAYLIST_TITLE = 'Youtube Subscriptions'
2524
2525
2526 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
2527 IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
2528 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
2529 _FEED_NAME = 'history'
2530 _PLAYLIST_TITLE = 'Youtube History'
2531
2532
2533 class YoutubeTruncatedURLIE(InfoExtractor):
2534 IE_NAME = 'youtube:truncated_url'
2535 IE_DESC = False # Do not list
2536 _VALID_URL = r'''(?x)
2537 (?:https?://)?
2538 (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
2539 (?:watch\?(?:
2540 feature=[a-z_]+|
2541 annotation_id=annotation_[^&]+|
2542 x-yt-cl=[0-9]+|
2543 hl=[^&]*|
2544 t=[0-9]+
2545 )?
2546 |
2547 attribution_link\?a=[^&]+
2548 )
2549 $
2550 '''
2551
2552 _TESTS = [{
2553 'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
2554 'only_matching': True,
2555 }, {
2556 'url': 'https://www.youtube.com/watch?',
2557 'only_matching': True,
2558 }, {
2559 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
2560 'only_matching': True,
2561 }, {
2562 'url': 'https://www.youtube.com/watch?feature=foo',
2563 'only_matching': True,
2564 }, {
2565 'url': 'https://www.youtube.com/watch?hl=en-GB',
2566 'only_matching': True,
2567 }, {
2568 'url': 'https://www.youtube.com/watch?t=2372',
2569 'only_matching': True,
2570 }]
2571
2572 def _real_extract(self, url):
2573 raise ExtractorError(
2574 'Did you forget to quote the URL? Remember that & is a meta '
2575 'character in most shells, so you want to put the URL in quotes, '
2576 'like youtube-dl '
2577 '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
2578 ' or simply youtube-dl BaW_jenozKc .',
2579 expected=True)
2580
2581
2582 class YoutubeTruncatedIDIE(InfoExtractor):
2583 IE_NAME = 'youtube:truncated_id'
2584 IE_DESC = False # Do not list
2585 _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
2586
2587 _TESTS = [{
2588 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
2589 'only_matching': True,
2590 }]
2591
2592 def _real_extract(self, url):
2593 video_id = self._match_id(url)
2594 raise ExtractorError(
2595 'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
2596 expected=True)