]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/youtube.py
2642ecfffbadeff3c7c184d52bf119328d5bbf60
[youtubedl] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 from __future__ import unicode_literals
4
5
6 import itertools
7 import json
8 import os.path
9 import re
10 import time
11 import traceback
12
13 from .common import InfoExtractor, SearchInfoExtractor
14 from .subtitles import SubtitlesInfoExtractor
15 from ..jsinterp import JSInterpreter
16 from ..swfinterp import SWFInterpreter
17 from ..utils import (
18 compat_chr,
19 compat_parse_qs,
20 compat_urllib_parse,
21 compat_urllib_request,
22 compat_urlparse,
23 compat_str,
24
25 clean_html,
26 get_element_by_id,
27 get_element_by_attribute,
28 ExtractorError,
29 int_or_none,
30 OnDemandPagedList,
31 unescapeHTML,
32 unified_strdate,
33 orderedSet,
34 uppercase_escape,
35 )
36
37
38 class YoutubeBaseInfoExtractor(InfoExtractor):
39 """Provide base functions for Youtube extractors"""
40 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
41 _TWOFACTOR_URL = 'https://accounts.google.com/SecondFactor'
42 _NETRC_MACHINE = 'youtube'
43 # If True it will raise an error if no login info is provided
44 _LOGIN_REQUIRED = False
45
46 def _set_language(self):
47 self._set_cookie('.youtube.com', 'PREF', 'f1=50000000&hl=en',
48 # YouTube sets the expire time to about two months
49 expire_time=time.time() + 60*24*3600)
50
51 def _login(self):
52 """
53 Attempt to log in to YouTube.
54 True is returned if successful or skipped.
55 False is returned if login failed.
56
57 If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
58 """
59 (username, password) = self._get_login_info()
60 # No authentication to be performed
61 if username is None:
62 if self._LOGIN_REQUIRED:
63 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
64 return True
65
66 login_page = self._download_webpage(
67 self._LOGIN_URL, None,
68 note='Downloading login page',
69 errnote='unable to fetch login page', fatal=False)
70 if login_page is False:
71 return
72
73 galx = self._search_regex(r'(?s)<input.+?name="GALX".+?value="(.+?)"',
74 login_page, 'Login GALX parameter')
75
76 # Log in
77 login_form_strs = {
78 'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
79 'Email': username,
80 'GALX': galx,
81 'Passwd': password,
82
83 'PersistentCookie': 'yes',
84 '_utf8': '霱',
85 'bgresponse': 'js_disabled',
86 'checkConnection': '',
87 'checkedDomains': 'youtube',
88 'dnConn': '',
89 'pstMsg': '0',
90 'rmShown': '1',
91 'secTok': '',
92 'signIn': 'Sign in',
93 'timeStmp': '',
94 'service': 'youtube',
95 'uilel': '3',
96 'hl': 'en_US',
97 }
98
99 # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
100 # chokes on unicode
101 login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
102 login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
103
104 req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
105 login_results = self._download_webpage(
106 req, None,
107 note='Logging in', errnote='unable to log in', fatal=False)
108 if login_results is False:
109 return False
110
111 if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
112 raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
113
114 # Two-Factor
115 # TODO add SMS and phone call support - these require making a request and then prompting the user
116
117 if re.search(r'(?i)<form[^>]* id="gaia_secondfactorform"', login_results) is not None:
118 tfa_code = self._get_tfa_info()
119
120 if tfa_code is None:
121 self._downloader.report_warning('Two-factor authentication required. Provide it with --twofactor <code>')
122 self._downloader.report_warning('(Note that only TOTP (Google Authenticator App) codes work at this time.)')
123 return False
124
125 # Unlike the first login form, secTok and timeStmp are both required for the TFA form
126
127 match = re.search(r'id="secTok"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
128 if match is None:
129 self._downloader.report_warning('Failed to get secTok - did the page structure change?')
130 secTok = match.group(1)
131 match = re.search(r'id="timeStmp"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
132 if match is None:
133 self._downloader.report_warning('Failed to get timeStmp - did the page structure change?')
134 timeStmp = match.group(1)
135
136 tfa_form_strs = {
137 'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
138 'smsToken': '',
139 'smsUserPin': tfa_code,
140 'smsVerifyPin': 'Verify',
141
142 'PersistentCookie': 'yes',
143 'checkConnection': '',
144 'checkedDomains': 'youtube',
145 'pstMsg': '1',
146 'secTok': secTok,
147 'timeStmp': timeStmp,
148 'service': 'youtube',
149 'hl': 'en_US',
150 }
151 tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in tfa_form_strs.items())
152 tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii')
153
154 tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
155 tfa_results = self._download_webpage(
156 tfa_req, None,
157 note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
158
159 if tfa_results is False:
160 return False
161
162 if re.search(r'(?i)<form[^>]* id="gaia_secondfactorform"', tfa_results) is not None:
163 self._downloader.report_warning('Two-factor code expired. Please try again, or use a one-use backup code instead.')
164 return False
165 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:
166 self._downloader.report_warning('unable to log in - did the page structure change?')
167 return False
168 if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
169 self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
170 return False
171
172 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
173 self._downloader.report_warning('unable to log in: bad username or password')
174 return False
175 return True
176
177 def _real_initialize(self):
178 if self._downloader is None:
179 return
180 self._set_language()
181 if not self._login():
182 return
183
184
185 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
186 IE_DESC = 'YouTube.com'
187 _VALID_URL = r"""(?x)^
188 (
189 (?:https?://|//) # http(s):// or protocol-independent URL
190 (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
191 (?:www\.)?deturl\.com/www\.youtube\.com/|
192 (?:www\.)?pwnyoutube\.com/|
193 (?:www\.)?yourepeat\.com/|
194 tube\.majestyc\.net/|
195 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
196 (?:.*?\#/)? # handle anchor (#/) redirect urls
197 (?: # the various things that can precede the ID:
198 (?:(?:v|embed|e)/(?!videoseries)) # v/ or embed/ or e/
199 |(?: # or the v= param in all its forms
200 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
201 (?:\?|\#!?) # the params delimiter ? or # or #!
202 (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
203 v=
204 )
205 ))
206 |youtu\.be/ # just youtu.be/xxxx
207 |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
208 )
209 )? # all until now is optional -> you can pass the naked ID
210 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
211 (?!.*?&list=) # combined list/video URLs are handled by the playlist IE
212 (?(1).+)? # if we found the ID, everything can follow
213 $"""
214 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
215 _formats = {
216 '5': {'ext': 'flv', 'width': 400, 'height': 240},
217 '6': {'ext': 'flv', 'width': 450, 'height': 270},
218 '13': {'ext': '3gp'},
219 '17': {'ext': '3gp', 'width': 176, 'height': 144},
220 '18': {'ext': 'mp4', 'width': 640, 'height': 360},
221 '22': {'ext': 'mp4', 'width': 1280, 'height': 720},
222 '34': {'ext': 'flv', 'width': 640, 'height': 360},
223 '35': {'ext': 'flv', 'width': 854, 'height': 480},
224 '36': {'ext': '3gp', 'width': 320, 'height': 240},
225 '37': {'ext': 'mp4', 'width': 1920, 'height': 1080},
226 '38': {'ext': 'mp4', 'width': 4096, 'height': 3072},
227 '43': {'ext': 'webm', 'width': 640, 'height': 360},
228 '44': {'ext': 'webm', 'width': 854, 'height': 480},
229 '45': {'ext': 'webm', 'width': 1280, 'height': 720},
230 '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
231
232
233 # 3d videos
234 '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'preference': -20},
235 '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'preference': -20},
236 '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'preference': -20},
237 '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'preference': -20},
238 '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'preference': -20},
239 '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'preference': -20},
240 '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'preference': -20},
241
242 # Apple HTTP Live Streaming
243 '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
244 '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'preference': -10},
245 '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'preference': -10},
246 '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'preference': -10},
247 '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'preference': -10},
248 '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
249 '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'preference': -10},
250
251 # DASH mp4 video
252 '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
253 '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
254 '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
255 '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
256 '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
257 '138': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
258 '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
259 '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
260 '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
261 '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
262 '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
263
264 # Dash mp4 audio
265 '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
266 '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 128, 'preference': -50},
267 '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50},
268
269 # Dash webm
270 '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
271 '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
272 '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
273 '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
274 '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
275 '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
276 '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'VP9'},
277 '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
278 '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
279 '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
280 '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
281 '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
282 '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
283 '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
284 '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
285 '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
286 '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
287 '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
288 '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'VP9'},
289
290 # Dash webm audio
291 '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
292 '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
293
294 # Dash webm audio with opus inside
295 '249': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
296 '250': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
297 '251': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
298
299 # RTMP (unnamed)
300 '_rtmp': {'protocol': 'rtmp'},
301 }
302
303 IE_NAME = 'youtube'
304 _TESTS = [
305 {
306 'url': 'http://www.youtube.com/watch?v=BaW_jenozKc',
307 'info_dict': {
308 'id': 'BaW_jenozKc',
309 'ext': 'mp4',
310 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
311 'uploader': 'Philipp Hagemeister',
312 'uploader_id': 'phihag',
313 'upload_date': '20121002',
314 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
315 'categories': ['Science & Technology'],
316 'like_count': int,
317 'dislike_count': int,
318 }
319 },
320 {
321 'url': 'http://www.youtube.com/watch?v=UxxajLWwzqY',
322 'note': 'Test generic use_cipher_signature video (#897)',
323 'info_dict': {
324 'id': 'UxxajLWwzqY',
325 'ext': 'mp4',
326 'upload_date': '20120506',
327 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
328 'description': 'md5:fea86fda2d5a5784273df5c7cc994d9f',
329 'uploader': 'Icona Pop',
330 'uploader_id': 'IconaPop',
331 }
332 },
333 {
334 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
335 'note': 'Test VEVO video with age protection (#956)',
336 'info_dict': {
337 'id': '07FYdnEawAQ',
338 'ext': 'mp4',
339 'upload_date': '20130703',
340 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
341 'description': 'md5:64249768eec3bc4276236606ea996373',
342 'uploader': 'justintimberlakeVEVO',
343 'uploader_id': 'justintimberlakeVEVO',
344 }
345 },
346 {
347 'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
348 'note': 'Embed-only video (#1746)',
349 'info_dict': {
350 'id': 'yZIXLfi8CZQ',
351 'ext': 'mp4',
352 'upload_date': '20120608',
353 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
354 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
355 'uploader': 'SET India',
356 'uploader_id': 'setindia'
357 }
358 },
359 {
360 'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',
361 'note': '256k DASH audio (format 141) via DASH manifest',
362 'info_dict': {
363 'id': 'a9LDPn-MO4I',
364 'ext': 'm4a',
365 'upload_date': '20121002',
366 'uploader_id': '8KVIDEO',
367 'description': '',
368 'uploader': '8KVIDEO',
369 'title': 'UHDTV TEST 8K VIDEO.mp4'
370 },
371 'params': {
372 'youtube_include_dash_manifest': True,
373 'format': '141',
374 },
375 },
376 # DASH manifest with encrypted signature
377 {
378 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
379 'info_dict': {
380 'id': 'IB3lcPjvWLA',
381 'ext': 'm4a',
382 'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
383 'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
384 'uploader': 'AfrojackVEVO',
385 'uploader_id': 'AfrojackVEVO',
386 'upload_date': '20131011',
387 },
388 'params': {
389 'youtube_include_dash_manifest': True,
390 'format': '141',
391 },
392 },
393 # Controversy video
394 {
395 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
396 'info_dict': {
397 'id': 'T4XJQO3qol8',
398 'ext': 'mp4',
399 'upload_date': '20100909',
400 'uploader': 'The Amazing Atheist',
401 'uploader_id': 'TheAmazingAtheist',
402 'title': 'Burning Everyone\'s Koran',
403 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
404 }
405 },
406 # Normal age-gate video (No vevo, embed allowed)
407 {
408 'url': 'http://youtube.com/watch?v=HtVdAasjOgU',
409 'info_dict': {
410 'id': 'HtVdAasjOgU',
411 'ext': 'mp4',
412 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
413 'description': 'md5:eca57043abae25130f58f655ad9a7771',
414 'uploader': 'The Witcher',
415 'uploader_id': 'WitcherGame',
416 'upload_date': '20140605',
417 },
418 },
419 ]
420
421 def __init__(self, *args, **kwargs):
422 super(YoutubeIE, self).__init__(*args, **kwargs)
423 self._player_cache = {}
424
425 def report_video_info_webpage_download(self, video_id):
426 """Report attempt to download video info webpage."""
427 self.to_screen('%s: Downloading video info webpage' % video_id)
428
429 def report_information_extraction(self, video_id):
430 """Report attempt to extract video information."""
431 self.to_screen('%s: Extracting video information' % video_id)
432
433 def report_unavailable_format(self, video_id, format):
434 """Report extracted video URL."""
435 self.to_screen('%s: Format %s not available' % (video_id, format))
436
437 def report_rtmp_download(self):
438 """Indicate the download will use the RTMP protocol."""
439 self.to_screen('RTMP download detected')
440
441 def _signature_cache_id(self, example_sig):
442 """ Return a string representation of a signature """
443 return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
444
445 def _extract_signature_function(self, video_id, player_url, example_sig):
446 id_m = re.match(
447 r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.(?P<ext>[a-z]+)$',
448 player_url)
449 if not id_m:
450 raise ExtractorError('Cannot identify player %r' % player_url)
451 player_type = id_m.group('ext')
452 player_id = id_m.group('id')
453
454 # Read from filesystem cache
455 func_id = '%s_%s_%s' % (
456 player_type, player_id, self._signature_cache_id(example_sig))
457 assert os.path.basename(func_id) == func_id
458
459 cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
460 if cache_spec is not None:
461 return lambda s: ''.join(s[i] for i in cache_spec)
462
463 if player_type == 'js':
464 code = self._download_webpage(
465 player_url, video_id,
466 note='Downloading %s player %s' % (player_type, player_id),
467 errnote='Download of %s failed' % player_url)
468 res = self._parse_sig_js(code)
469 elif player_type == 'swf':
470 urlh = self._request_webpage(
471 player_url, video_id,
472 note='Downloading %s player %s' % (player_type, player_id),
473 errnote='Download of %s failed' % player_url)
474 code = urlh.read()
475 res = self._parse_sig_swf(code)
476 else:
477 assert False, 'Invalid player type %r' % player_type
478
479 if cache_spec is None:
480 test_string = ''.join(map(compat_chr, range(len(example_sig))))
481 cache_res = res(test_string)
482 cache_spec = [ord(c) for c in cache_res]
483
484 self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
485 return res
486
487 def _print_sig_code(self, func, example_sig):
488 def gen_sig_code(idxs):
489 def _genslice(start, end, step):
490 starts = '' if start == 0 else str(start)
491 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
492 steps = '' if step == 1 else (':%d' % step)
493 return 's[%s%s%s]' % (starts, ends, steps)
494
495 step = None
496 start = '(Never used)' # Quelch pyflakes warnings - start will be
497 # set as soon as step is set
498 for i, prev in zip(idxs[1:], idxs[:-1]):
499 if step is not None:
500 if i - prev == step:
501 continue
502 yield _genslice(start, prev, step)
503 step = None
504 continue
505 if i - prev in [-1, 1]:
506 step = i - prev
507 start = prev
508 continue
509 else:
510 yield 's[%d]' % prev
511 if step is None:
512 yield 's[%d]' % i
513 else:
514 yield _genslice(start, i, step)
515
516 test_string = ''.join(map(compat_chr, range(len(example_sig))))
517 cache_res = func(test_string)
518 cache_spec = [ord(c) for c in cache_res]
519 expr_code = ' + '.join(gen_sig_code(cache_spec))
520 signature_id_tuple = '(%s)' % (
521 ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
522 code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
523 ' return %s\n') % (signature_id_tuple, expr_code)
524 self.to_screen('Extracted signature function:\n' + code)
525
526 def _parse_sig_js(self, jscode):
527 funcname = self._search_regex(
528 r'\.sig\|\|([a-zA-Z0-9]+)\(', jscode,
529 'Initial JS player signature function name')
530
531 jsi = JSInterpreter(jscode)
532 initial_function = jsi.extract_function(funcname)
533 return lambda s: initial_function([s])
534
535 def _parse_sig_swf(self, file_contents):
536 swfi = SWFInterpreter(file_contents)
537 TARGET_CLASSNAME = 'SignatureDecipher'
538 searched_class = swfi.extract_class(TARGET_CLASSNAME)
539 initial_function = swfi.extract_function(searched_class, 'decipher')
540 return lambda s: initial_function([s])
541
542 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
543 """Turn the encrypted s field into a working signature"""
544
545 if player_url is None:
546 raise ExtractorError('Cannot decrypt signature without player_url')
547
548 if player_url.startswith('//'):
549 player_url = 'https:' + player_url
550 try:
551 player_id = (player_url, self._signature_cache_id(s))
552 if player_id not in self._player_cache:
553 func = self._extract_signature_function(
554 video_id, player_url, s
555 )
556 self._player_cache[player_id] = func
557 func = self._player_cache[player_id]
558 if self._downloader.params.get('youtube_print_sig_code'):
559 self._print_sig_code(func, s)
560 return func(s)
561 except Exception as e:
562 tb = traceback.format_exc()
563 raise ExtractorError(
564 'Signature extraction failed: ' + tb, cause=e)
565
566 def _get_available_subtitles(self, video_id, webpage):
567 try:
568 sub_list = self._download_webpage(
569 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
570 video_id, note=False)
571 except ExtractorError as err:
572 self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
573 return {}
574 lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
575
576 sub_lang_list = {}
577 for l in lang_list:
578 lang = l[1]
579 if lang in sub_lang_list:
580 continue
581 params = compat_urllib_parse.urlencode({
582 'lang': lang,
583 'v': video_id,
584 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
585 'name': unescapeHTML(l[0]).encode('utf-8'),
586 })
587 url = 'https://www.youtube.com/api/timedtext?' + params
588 sub_lang_list[lang] = url
589 if not sub_lang_list:
590 self._downloader.report_warning('video doesn\'t have subtitles')
591 return {}
592 return sub_lang_list
593
594 def _get_available_automatic_caption(self, video_id, webpage):
595 """We need the webpage for getting the captions url, pass it as an
596 argument to speed up the process."""
597 sub_format = self._downloader.params.get('subtitlesformat', 'srt')
598 self.to_screen('%s: Looking for automatic captions' % video_id)
599 mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
600 err_msg = 'Couldn\'t find automatic captions for %s' % video_id
601 if mobj is None:
602 self._downloader.report_warning(err_msg)
603 return {}
604 player_config = json.loads(mobj.group(1))
605 try:
606 args = player_config['args']
607 caption_url = args['ttsurl']
608 timestamp = args['timestamp']
609 # We get the available subtitles
610 list_params = compat_urllib_parse.urlencode({
611 'type': 'list',
612 'tlangs': 1,
613 'asrs': 1,
614 })
615 list_url = caption_url + '&' + list_params
616 caption_list = self._download_xml(list_url, video_id)
617 original_lang_node = caption_list.find('track')
618 if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr':
619 self._downloader.report_warning('Video doesn\'t have automatic captions')
620 return {}
621 original_lang = original_lang_node.attrib['lang_code']
622
623 sub_lang_list = {}
624 for lang_node in caption_list.findall('target'):
625 sub_lang = lang_node.attrib['lang_code']
626 params = compat_urllib_parse.urlencode({
627 'lang': original_lang,
628 'tlang': sub_lang,
629 'fmt': sub_format,
630 'ts': timestamp,
631 'kind': 'asr',
632 })
633 sub_lang_list[sub_lang] = caption_url + '&' + params
634 return sub_lang_list
635 # An extractor error can be raise by the download process if there are
636 # no automatic captions but there are subtitles
637 except (KeyError, ExtractorError):
638 self._downloader.report_warning(err_msg)
639 return {}
640
641 @classmethod
642 def extract_id(cls, url):
643 mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
644 if mobj is None:
645 raise ExtractorError('Invalid URL: %s' % url)
646 video_id = mobj.group(2)
647 return video_id
648
649 def _extract_from_m3u8(self, manifest_url, video_id):
650 url_map = {}
651
652 def _get_urls(_manifest):
653 lines = _manifest.split('\n')
654 urls = filter(lambda l: l and not l.startswith('#'),
655 lines)
656 return urls
657 manifest = self._download_webpage(manifest_url, video_id, 'Downloading formats manifest')
658 formats_urls = _get_urls(manifest)
659 for format_url in formats_urls:
660 itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
661 url_map[itag] = format_url
662 return url_map
663
664 def _extract_annotations(self, video_id):
665 url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
666 return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
667
668 def _real_extract(self, url):
669 proto = (
670 'http' if self._downloader.params.get('prefer_insecure', False)
671 else 'https')
672
673 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
674 mobj = re.search(self._NEXT_URL_RE, url)
675 if mobj:
676 url = proto + '://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
677 video_id = self.extract_id(url)
678
679 # Get video webpage
680 url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
681 video_webpage = self._download_webpage(url, video_id)
682
683 # Attempt to extract SWF player URL
684 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
685 if mobj is not None:
686 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
687 else:
688 player_url = None
689
690 # Get video info
691 if re.search(r'player-age-gate-content">', video_webpage) is not None:
692 age_gate = True
693 # We simulate the access to the video from www.youtube.com/v/{video_id}
694 # this can be viewed without login into Youtube
695 data = compat_urllib_parse.urlencode({
696 'video_id': video_id,
697 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
698 'sts': self._search_regex(
699 r'"sts"\s*:\s*(\d+)', video_webpage, 'sts', default=''),
700 })
701 video_info_url = proto + '://www.youtube.com/get_video_info?' + data
702 video_info_webpage = self._download_webpage(
703 video_info_url, video_id,
704 note='Refetching age-gated info webpage',
705 errnote='unable to download video info webpage')
706 video_info = compat_parse_qs(video_info_webpage)
707 else:
708 age_gate = False
709 try:
710 # Try looking directly into the video webpage
711 mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
712 if not mobj:
713 raise ValueError('Could not find ytplayer.config') # caught below
714 json_code = uppercase_escape(mobj.group(1))
715 ytplayer_config = json.loads(json_code)
716 args = ytplayer_config['args']
717 # Convert to the same format returned by compat_parse_qs
718 video_info = dict((k, [v]) for k, v in args.items())
719 if 'url_encoded_fmt_stream_map' not in args:
720 raise ValueError('No stream_map present') # caught below
721 except ValueError:
722 # We fallback to the get_video_info pages (used by the embed page)
723 self.report_video_info_webpage_download(video_id)
724 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
725 video_info_url = (proto + '://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
726 % (video_id, el_type))
727 video_info_webpage = self._download_webpage(video_info_url,
728 video_id, note=False,
729 errnote='unable to download video info webpage')
730 video_info = compat_parse_qs(video_info_webpage)
731 if 'token' in video_info:
732 break
733 if 'token' not in video_info:
734 if 'reason' in video_info:
735 raise ExtractorError(
736 'YouTube said: %s' % video_info['reason'][0],
737 expected=True, video_id=video_id)
738 else:
739 raise ExtractorError(
740 '"token" parameter not in video info for unknown reason',
741 video_id=video_id)
742
743 if 'view_count' in video_info:
744 view_count = int(video_info['view_count'][0])
745 else:
746 view_count = None
747
748 # Check for "rental" videos
749 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
750 raise ExtractorError('"rental" videos not supported')
751
752 # Start extracting information
753 self.report_information_extraction(video_id)
754
755 # uploader
756 if 'author' not in video_info:
757 raise ExtractorError('Unable to extract uploader name')
758 video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
759
760 # uploader_id
761 video_uploader_id = None
762 mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
763 if mobj is not None:
764 video_uploader_id = mobj.group(1)
765 else:
766 self._downloader.report_warning('unable to extract uploader nickname')
767
768 # title
769 if 'title' in video_info:
770 video_title = video_info['title'][0]
771 else:
772 self._downloader.report_warning('Unable to extract video title')
773 video_title = '_'
774
775 # thumbnail image
776 # We try first to get a high quality image:
777 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
778 video_webpage, re.DOTALL)
779 if m_thumb is not None:
780 video_thumbnail = m_thumb.group(1)
781 elif 'thumbnail_url' not in video_info:
782 self._downloader.report_warning('unable to extract video thumbnail')
783 video_thumbnail = None
784 else: # don't panic if we can't find it
785 video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
786
787 # upload date
788 upload_date = None
789 mobj = re.search(r'(?s)id="eow-date.*?>(.*?)</span>', video_webpage)
790 if mobj is None:
791 mobj = re.search(
792 r'(?s)id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live) on (.*?)</strong>',
793 video_webpage)
794 if mobj is not None:
795 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
796 upload_date = unified_strdate(upload_date)
797
798 m_cat_container = self._search_regex(
799 r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
800 video_webpage, 'categories', fatal=False)
801 if m_cat_container:
802 category = self._html_search_regex(
803 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
804 default=None)
805 video_categories = None if category is None else [category]
806 else:
807 video_categories = None
808
809 # description
810 video_description = get_element_by_id("eow-description", video_webpage)
811 if video_description:
812 video_description = re.sub(r'''(?x)
813 <a\s+
814 (?:[a-zA-Z-]+="[^"]+"\s+)*?
815 title="([^"]+)"\s+
816 (?:[a-zA-Z-]+="[^"]+"\s+)*?
817 class="yt-uix-redirect-link"\s*>
818 [^<]+
819 </a>
820 ''', r'\1', video_description)
821 video_description = clean_html(video_description)
822 else:
823 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
824 if fd_mobj:
825 video_description = unescapeHTML(fd_mobj.group(1))
826 else:
827 video_description = ''
828
829 def _extract_count(count_name):
830 count = self._search_regex(
831 r'id="watch-%s"[^>]*>.*?([\d,]+)\s*</span>' % re.escape(count_name),
832 video_webpage, count_name, default=None)
833 if count is not None:
834 return int(count.replace(',', ''))
835 return None
836 like_count = _extract_count('like')
837 dislike_count = _extract_count('dislike')
838
839 # subtitles
840 video_subtitles = self.extract_subtitles(video_id, video_webpage)
841
842 if self._downloader.params.get('listsubtitles', False):
843 self._list_available_subtitles(video_id, video_webpage)
844 return
845
846 if 'length_seconds' not in video_info:
847 self._downloader.report_warning('unable to extract video duration')
848 video_duration = None
849 else:
850 video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]))
851
852 # annotations
853 video_annotations = None
854 if self._downloader.params.get('writeannotations', False):
855 video_annotations = self._extract_annotations(video_id)
856
857 def _map_to_format_list(urlmap):
858 formats = []
859 for itag, video_real_url in urlmap.items():
860 dct = {
861 'format_id': itag,
862 'url': video_real_url,
863 'player_url': player_url,
864 }
865 if itag in self._formats:
866 dct.update(self._formats[itag])
867 formats.append(dct)
868 return formats
869
870 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
871 self.report_rtmp_download()
872 formats = [{
873 'format_id': '_rtmp',
874 'protocol': 'rtmp',
875 'url': video_info['conn'][0],
876 'player_url': player_url,
877 }]
878 elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
879 encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
880 if 'rtmpe%3Dyes' in encoded_url_map:
881 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
882 url_map = {}
883 for url_data_str in encoded_url_map.split(','):
884 url_data = compat_parse_qs(url_data_str)
885 if 'itag' not in url_data or 'url' not in url_data:
886 continue
887 format_id = url_data['itag'][0]
888 url = url_data['url'][0]
889
890 if 'sig' in url_data:
891 url += '&signature=' + url_data['sig'][0]
892 elif 's' in url_data:
893 encrypted_sig = url_data['s'][0]
894
895 if not age_gate:
896 jsplayer_url_json = self._search_regex(
897 r'"assets":.+?"js":\s*("[^"]+")',
898 video_webpage, 'JS player URL')
899 player_url = json.loads(jsplayer_url_json)
900 if player_url is None:
901 player_url_json = self._search_regex(
902 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
903 video_webpage, 'age gate player URL')
904 player_url = json.loads(player_url_json)
905
906 if self._downloader.params.get('verbose'):
907 if player_url is None:
908 player_version = 'unknown'
909 player_desc = 'unknown'
910 else:
911 if player_url.endswith('swf'):
912 player_version = self._search_regex(
913 r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
914 'flash player', fatal=False)
915 player_desc = 'flash player %s' % player_version
916 else:
917 player_version = self._search_regex(
918 r'html5player-([^/]+?)(?:/html5player)?\.js',
919 player_url,
920 'html5 player', fatal=False)
921 player_desc = 'html5 player %s' % player_version
922
923 parts_sizes = self._signature_cache_id(encrypted_sig)
924 self.to_screen('{%s} signature length %s, %s' %
925 (format_id, parts_sizes, player_desc))
926
927 signature = self._decrypt_signature(
928 encrypted_sig, video_id, player_url, age_gate)
929 url += '&signature=' + signature
930 if 'ratebypass' not in url:
931 url += '&ratebypass=yes'
932 url_map[format_id] = url
933 formats = _map_to_format_list(url_map)
934 elif video_info.get('hlsvp'):
935 manifest_url = video_info['hlsvp'][0]
936 url_map = self._extract_from_m3u8(manifest_url, video_id)
937 formats = _map_to_format_list(url_map)
938 else:
939 raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
940
941 # Look for the DASH manifest
942 if self._downloader.params.get('youtube_include_dash_manifest', True):
943 try:
944 # The DASH manifest used needs to be the one from the original video_webpage.
945 # The one found in get_video_info seems to be using different signatures.
946 # However, in the case of an age restriction there won't be any embedded dashmpd in the video_webpage.
947 # Luckily, it seems, this case uses some kind of default signature (len == 86), so the
948 # combination of get_video_info and the _static_decrypt_signature() decryption fallback will work here.
949 dash_manifest_url = video_info.get('dashmpd')[0]
950
951 def decrypt_sig(mobj):
952 s = mobj.group(1)
953 dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
954 return '/signature/%s' % dec_s
955 dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
956 dash_doc = self._download_xml(
957 dash_manifest_url, video_id,
958 note='Downloading DASH manifest',
959 errnote='Could not download DASH manifest')
960 for r in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
961 url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
962 if url_el is None:
963 continue
964 format_id = r.attrib['id']
965 video_url = url_el.text
966 filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
967 f = {
968 'format_id': format_id,
969 'url': video_url,
970 'width': int_or_none(r.attrib.get('width')),
971 'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
972 'asr': int_or_none(r.attrib.get('audioSamplingRate')),
973 'filesize': filesize,
974 }
975 try:
976 existing_format = next(
977 fo for fo in formats
978 if fo['format_id'] == format_id)
979 except StopIteration:
980 f.update(self._formats.get(format_id, {}))
981 formats.append(f)
982 else:
983 existing_format.update(f)
984
985 except (ExtractorError, KeyError) as e:
986 self.report_warning('Skipping DASH manifest: %r' % e, video_id)
987
988 self._sort_formats(formats)
989
990 return {
991 'id': video_id,
992 'uploader': video_uploader,
993 'uploader_id': video_uploader_id,
994 'upload_date': upload_date,
995 'title': video_title,
996 'thumbnail': video_thumbnail,
997 'description': video_description,
998 'categories': video_categories,
999 'subtitles': video_subtitles,
1000 'duration': video_duration,
1001 'age_limit': 18 if age_gate else 0,
1002 'annotations': video_annotations,
1003 'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
1004 'view_count': view_count,
1005 'like_count': like_count,
1006 'dislike_count': dislike_count,
1007 'formats': formats,
1008 }
1009
1010
1011 class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
1012 IE_DESC = 'YouTube.com playlists'
1013 _VALID_URL = r"""(?x)(?:
1014 (?:https?://)?
1015 (?:\w+\.)?
1016 youtube\.com/
1017 (?:
1018 (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
1019 \? (?:.*?&)*? (?:p|a|list)=
1020 | p/
1021 )
1022 (
1023 (?:PL|LL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,}
1024 # Top tracks, they can also include dots
1025 |(?:MC)[\w\.]*
1026 )
1027 .*
1028 |
1029 ((?:PL|LL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
1030 )"""
1031 _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
1032 _MORE_PAGES_INDICATOR = r'data-link-type="next"'
1033 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
1034 IE_NAME = 'youtube:playlist'
1035 _TESTS = [{
1036 'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1037 'info_dict': {
1038 'title': 'ytdl test PL',
1039 'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1040 },
1041 'playlist_count': 3,
1042 }, {
1043 'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
1044 'info_dict': {
1045 'title': 'YDL_Empty_List',
1046 },
1047 'playlist_count': 0,
1048 }, {
1049 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
1050 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1051 'info_dict': {
1052 'title': '29C3: Not my department',
1053 },
1054 'playlist_count': 95,
1055 }, {
1056 'note': 'issue #673',
1057 'url': 'PLBB231211A4F62143',
1058 'info_dict': {
1059 'title': '[OLD]Team Fortress 2 (Class-based LP)',
1060 },
1061 'playlist_mincount': 26,
1062 }, {
1063 'note': 'Large playlist',
1064 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
1065 'info_dict': {
1066 'title': 'Uploads from Cauchemar',
1067 },
1068 'playlist_mincount': 799,
1069 }, {
1070 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1071 'info_dict': {
1072 'title': 'YDL_safe_search',
1073 },
1074 'playlist_count': 2,
1075 }, {
1076 'note': 'embedded',
1077 'url': 'http://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1078 'playlist_count': 4,
1079 'info_dict': {
1080 'title': 'JODA15',
1081 }
1082 }, {
1083 'note': 'Embedded SWF player',
1084 'url': 'http://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
1085 'playlist_count': 4,
1086 'info_dict': {
1087 'title': 'JODA7',
1088 }
1089 }]
1090
1091 def _real_initialize(self):
1092 self._login()
1093
1094 def _ids_to_results(self, ids):
1095 return [
1096 self.url_result(vid_id, 'Youtube', video_id=vid_id)
1097 for vid_id in ids]
1098
1099 def _extract_mix(self, playlist_id):
1100 # The mixes are generated from a a single video
1101 # the id of the playlist is just 'RD' + video_id
1102 url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
1103 webpage = self._download_webpage(
1104 url, playlist_id, 'Downloading Youtube mix')
1105 search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
1106 title_span = (
1107 search_title('playlist-title') or
1108 search_title('title long-title') or
1109 search_title('title'))
1110 title = clean_html(title_span)
1111 ids = orderedSet(re.findall(
1112 r'''(?xs)data-video-username=".*?".*?
1113 href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
1114 webpage))
1115 url_results = self._ids_to_results(ids)
1116
1117 return self.playlist_result(url_results, playlist_id, title)
1118
1119 def _real_extract(self, url):
1120 # Extract playlist id
1121 mobj = re.match(self._VALID_URL, url)
1122 if mobj is None:
1123 raise ExtractorError('Invalid URL: %s' % url)
1124 playlist_id = mobj.group(1) or mobj.group(2)
1125
1126 # Check if it's a video-specific URL
1127 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
1128 if 'v' in query_dict:
1129 video_id = query_dict['v'][0]
1130 if self._downloader.params.get('noplaylist'):
1131 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1132 return self.url_result(video_id, 'Youtube', video_id=video_id)
1133 else:
1134 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
1135
1136 if playlist_id.startswith('RD'):
1137 # Mixes require a custom extraction process
1138 return self._extract_mix(playlist_id)
1139 if playlist_id.startswith('TL'):
1140 raise ExtractorError('For downloading YouTube.com top lists, use '
1141 'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
1142
1143 url = self._TEMPLATE_URL % playlist_id
1144 page = self._download_webpage(url, playlist_id)
1145 more_widget_html = content_html = page
1146
1147 # Check if the playlist exists or is private
1148 if re.search(r'<div class="yt-alert-message">[^<]*?(The|This) playlist (does not exist|is private)[^<]*?</div>', page) is not None:
1149 raise ExtractorError(
1150 'The playlist doesn\'t exist or is private, use --username or '
1151 '--netrc to access it.',
1152 expected=True)
1153
1154 # Extract the video ids from the playlist pages
1155 ids = []
1156
1157 for page_num in itertools.count(1):
1158 matches = re.finditer(self._VIDEO_RE, content_html)
1159 # We remove the duplicates and the link with index 0
1160 # (it's not the first video of the playlist)
1161 new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
1162 ids.extend(new_ids)
1163
1164 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
1165 if not mobj:
1166 break
1167
1168 more = self._download_json(
1169 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
1170 'Downloading page #%s' % page_num,
1171 transform_source=uppercase_escape)
1172 content_html = more['content_html']
1173 more_widget_html = more['load_more_widget_html']
1174
1175 playlist_title = self._html_search_regex(
1176 r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
1177 page, 'title')
1178
1179 url_results = self._ids_to_results(ids)
1180 return self.playlist_result(url_results, playlist_id, playlist_title)
1181
1182
1183 class YoutubeTopListIE(YoutubePlaylistIE):
1184 IE_NAME = 'youtube:toplist'
1185 IE_DESC = ('YouTube.com top lists, "yttoplist:{channel}:{list title}"'
1186 ' (Example: "yttoplist:music:Top Tracks")')
1187 _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
1188 _TESTS = [{
1189 'url': 'yttoplist:music:Trending',
1190 'playlist_mincount': 5,
1191 'skip': 'Only works for logged-in users',
1192 }]
1193
1194 def _real_extract(self, url):
1195 mobj = re.match(self._VALID_URL, url)
1196 channel = mobj.group('chann')
1197 title = mobj.group('title')
1198 query = compat_urllib_parse.urlencode({'title': title})
1199 channel_page = self._download_webpage(
1200 'https://www.youtube.com/%s' % channel, title)
1201 link = self._html_search_regex(
1202 r'''(?x)
1203 <a\s+href="([^"]+)".*?>\s*
1204 <span\s+class="branded-page-module-title-text">\s*
1205 <span[^>]*>.*?%s.*?</span>''' % re.escape(query),
1206 channel_page, 'list')
1207 url = compat_urlparse.urljoin('https://www.youtube.com/', link)
1208
1209 video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
1210 ids = []
1211 # sometimes the webpage doesn't contain the videos
1212 # retry until we get them
1213 for i in itertools.count(0):
1214 msg = 'Downloading Youtube mix'
1215 if i > 0:
1216 msg += ', retry #%d' % i
1217
1218 webpage = self._download_webpage(url, title, msg)
1219 ids = orderedSet(re.findall(video_re, webpage))
1220 if ids:
1221 break
1222 url_results = self._ids_to_results(ids)
1223 return self.playlist_result(url_results, playlist_title=title)
1224
1225
1226 class YoutubeChannelIE(InfoExtractor):
1227 IE_DESC = 'YouTube.com channels'
1228 _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
1229 _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
1230 _MORE_PAGES_URL = 'https://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
1231 IE_NAME = 'youtube:channel'
1232 _TESTS = [{
1233 'note': 'paginated channel',
1234 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
1235 'playlist_mincount': 91,
1236 }]
1237
1238 def extract_videos_from_page(self, page):
1239 ids_in_page = []
1240 for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
1241 if mobj.group(1) not in ids_in_page:
1242 ids_in_page.append(mobj.group(1))
1243 return ids_in_page
1244
1245 def _real_extract(self, url):
1246 # Extract channel id
1247 mobj = re.match(self._VALID_URL, url)
1248 if mobj is None:
1249 raise ExtractorError('Invalid URL: %s' % url)
1250
1251 # Download channel page
1252 channel_id = mobj.group(1)
1253 video_ids = []
1254 url = 'https://www.youtube.com/channel/%s/videos' % channel_id
1255 channel_page = self._download_webpage(url, channel_id)
1256 autogenerated = re.search(r'''(?x)
1257 class="[^"]*?(?:
1258 channel-header-autogenerated-label|
1259 yt-channel-title-autogenerated
1260 )[^"]*"''', channel_page) is not None
1261
1262 if autogenerated:
1263 # The videos are contained in a single page
1264 # the ajax pages can't be used, they are empty
1265 video_ids = self.extract_videos_from_page(channel_page)
1266 else:
1267 # Download all channel pages using the json-based channel_ajax query
1268 for pagenum in itertools.count(1):
1269 url = self._MORE_PAGES_URL % (pagenum, channel_id)
1270 page = self._download_json(
1271 url, channel_id, note='Downloading page #%s' % pagenum,
1272 transform_source=uppercase_escape)
1273
1274 ids_in_page = self.extract_videos_from_page(page['content_html'])
1275 video_ids.extend(ids_in_page)
1276
1277 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
1278 break
1279
1280 self._downloader.to_screen('[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
1281
1282 url_entries = [self.url_result(video_id, 'Youtube', video_id=video_id)
1283 for video_id in video_ids]
1284 return self.playlist_result(url_entries, channel_id)
1285
1286
1287 class YoutubeUserIE(InfoExtractor):
1288 IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
1289 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
1290 _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/users/%s'
1291 _GDATA_PAGE_SIZE = 50
1292 _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
1293 IE_NAME = 'youtube:user'
1294
1295 _TESTS = [{
1296 'url': 'https://www.youtube.com/user/TheLinuxFoundation',
1297 'playlist_mincount': 320,
1298 'info_dict': {
1299 'title': 'TheLinuxFoundation',
1300 }
1301 }, {
1302 'url': 'ytuser:phihag',
1303 'only_matching': True,
1304 }]
1305
1306 @classmethod
1307 def suitable(cls, url):
1308 # Don't return True if the url can be extracted with other youtube
1309 # extractor, the regex would is too permissive and it would match.
1310 other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
1311 if any(ie.suitable(url) for ie in other_ies):
1312 return False
1313 else:
1314 return super(YoutubeUserIE, cls).suitable(url)
1315
1316 def _real_extract(self, url):
1317 # Extract username
1318 mobj = re.match(self._VALID_URL, url)
1319 if mobj is None:
1320 raise ExtractorError('Invalid URL: %s' % url)
1321
1322 username = mobj.group(1)
1323
1324 # Download video ids using YouTube Data API. Result size per
1325 # query is limited (currently to 50 videos) so we need to query
1326 # page by page until there are no video ids - it means we got
1327 # all of them.
1328
1329 def download_page(pagenum):
1330 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1331
1332 gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
1333 page = self._download_webpage(
1334 gdata_url, username,
1335 'Downloading video ids from %d to %d' % (
1336 start_index, start_index + self._GDATA_PAGE_SIZE))
1337
1338 try:
1339 response = json.loads(page)
1340 except ValueError as err:
1341 raise ExtractorError('Invalid JSON in API response: ' + compat_str(err))
1342 if 'entry' not in response['feed']:
1343 return
1344
1345 # Extract video identifiers
1346 entries = response['feed']['entry']
1347 for entry in entries:
1348 title = entry['title']['$t']
1349 video_id = entry['id']['$t'].split('/')[-1]
1350 yield {
1351 '_type': 'url',
1352 'url': video_id,
1353 'ie_key': 'Youtube',
1354 'id': video_id,
1355 'title': title,
1356 }
1357 url_results = OnDemandPagedList(download_page, self._GDATA_PAGE_SIZE)
1358
1359 return self.playlist_result(url_results, playlist_title=username)
1360
1361
1362 class YoutubeSearchIE(SearchInfoExtractor):
1363 IE_DESC = 'YouTube.com searches'
1364 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1365 _MAX_RESULTS = 1000
1366 IE_NAME = 'youtube:search'
1367 _SEARCH_KEY = 'ytsearch'
1368
1369 def _get_n_results(self, query, n):
1370 """Get a specified number of results for a query"""
1371
1372 video_ids = []
1373 pagenum = 0
1374 limit = n
1375 PAGE_SIZE = 50
1376
1377 while (PAGE_SIZE * pagenum) < limit:
1378 result_url = self._API_URL % (
1379 compat_urllib_parse.quote_plus(query.encode('utf-8')),
1380 (PAGE_SIZE * pagenum) + 1)
1381 data_json = self._download_webpage(
1382 result_url, video_id='query "%s"' % query,
1383 note='Downloading page %s' % (pagenum + 1),
1384 errnote='Unable to download API page')
1385 data = json.loads(data_json)
1386 api_response = data['data']
1387
1388 if 'items' not in api_response:
1389 raise ExtractorError(
1390 '[youtube] No video results', expected=True)
1391
1392 new_ids = list(video['id'] for video in api_response['items'])
1393 video_ids += new_ids
1394
1395 limit = min(n, api_response['totalItems'])
1396 pagenum += 1
1397
1398 if len(video_ids) > n:
1399 video_ids = video_ids[:n]
1400 videos = [self.url_result(video_id, 'Youtube', video_id=video_id)
1401 for video_id in video_ids]
1402 return self.playlist_result(videos, query)
1403
1404
1405 class YoutubeSearchDateIE(YoutubeSearchIE):
1406 IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
1407 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
1408 _SEARCH_KEY = 'ytsearchdate'
1409 IE_DESC = 'YouTube.com searches, newest videos first'
1410
1411
1412 class YoutubeSearchURLIE(InfoExtractor):
1413 IE_DESC = 'YouTube.com search URLs'
1414 IE_NAME = 'youtube:search_url'
1415 _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?search_query=(?P<query>[^&]+)(?:[&]|$)'
1416 _TESTS = [{
1417 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
1418 'playlist_mincount': 5,
1419 'info_dict': {
1420 'title': 'youtube-dl test video',
1421 }
1422 }]
1423
1424 def _real_extract(self, url):
1425 mobj = re.match(self._VALID_URL, url)
1426 query = compat_urllib_parse.unquote_plus(mobj.group('query'))
1427
1428 webpage = self._download_webpage(url, query)
1429 result_code = self._search_regex(
1430 r'(?s)<ol class="item-section"(.*?)</ol>', webpage, 'result HTML')
1431
1432 part_codes = re.findall(
1433 r'(?s)<h3 class="yt-lockup-title">(.*?)</h3>', result_code)
1434 entries = []
1435 for part_code in part_codes:
1436 part_title = self._html_search_regex(
1437 [r'(?s)title="([^"]+)"', r'>([^<]+)</a>'], part_code, 'item title', fatal=False)
1438 part_url_snippet = self._html_search_regex(
1439 r'(?s)href="([^"]+)"', part_code, 'item URL')
1440 part_url = compat_urlparse.urljoin(
1441 'https://www.youtube.com/', part_url_snippet)
1442 entries.append({
1443 '_type': 'url',
1444 'url': part_url,
1445 'title': part_title,
1446 })
1447
1448 return {
1449 '_type': 'playlist',
1450 'entries': entries,
1451 'title': query,
1452 }
1453
1454
1455 class YoutubeShowIE(InfoExtractor):
1456 IE_DESC = 'YouTube.com (multi-season) shows'
1457 _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
1458 IE_NAME = 'youtube:show'
1459 _TESTS = [{
1460 'url': 'http://www.youtube.com/show/airdisasters',
1461 'playlist_mincount': 3,
1462 'info_dict': {
1463 'id': 'airdisasters',
1464 'title': 'Air Disasters',
1465 }
1466 }]
1467
1468 def _real_extract(self, url):
1469 mobj = re.match(self._VALID_URL, url)
1470 playlist_id = mobj.group('id')
1471 webpage = self._download_webpage(
1472 url, playlist_id, 'Downloading show webpage')
1473 # There's one playlist for each season of the show
1474 m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
1475 self.to_screen('%s: Found %s seasons' % (playlist_id, len(m_seasons)))
1476 entries = [
1477 self.url_result(
1478 'https://www.youtube.com' + season.group(1), 'YoutubePlaylist')
1479 for season in m_seasons
1480 ]
1481 title = self._og_search_title(webpage, fatal=False)
1482
1483 return {
1484 '_type': 'playlist',
1485 'id': playlist_id,
1486 'title': title,
1487 'entries': entries,
1488 }
1489
1490
1491 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
1492 """
1493 Base class for extractors that fetch info from
1494 http://www.youtube.com/feed_ajax
1495 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
1496 """
1497 _LOGIN_REQUIRED = True
1498 # use action_load_personal_feed instead of action_load_system_feed
1499 _PERSONAL_FEED = False
1500
1501 @property
1502 def _FEED_TEMPLATE(self):
1503 action = 'action_load_system_feed'
1504 if self._PERSONAL_FEED:
1505 action = 'action_load_personal_feed'
1506 return 'https://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
1507
1508 @property
1509 def IE_NAME(self):
1510 return 'youtube:%s' % self._FEED_NAME
1511
1512 def _real_initialize(self):
1513 self._login()
1514
1515 def _real_extract(self, url):
1516 feed_entries = []
1517 paging = 0
1518 for i in itertools.count(1):
1519 info = self._download_json(self._FEED_TEMPLATE % paging,
1520 '%s feed' % self._FEED_NAME,
1521 'Downloading page %s' % i)
1522 feed_html = info.get('feed_html') or info.get('content_html')
1523 load_more_widget_html = info.get('load_more_widget_html') or feed_html
1524 m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
1525 ids = orderedSet(m.group(1) for m in m_ids)
1526 feed_entries.extend(
1527 self.url_result(video_id, 'Youtube', video_id=video_id)
1528 for video_id in ids)
1529 mobj = re.search(
1530 r'data-uix-load-more-href="/?[^"]+paging=(?P<paging>\d+)',
1531 load_more_widget_html)
1532 if mobj is None:
1533 break
1534 paging = mobj.group('paging')
1535 return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
1536
1537
1538 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
1539 IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
1540 _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
1541 _FEED_NAME = 'recommended'
1542 _PLAYLIST_TITLE = 'Youtube Recommended videos'
1543
1544
1545 class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
1546 IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
1547 _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
1548 _FEED_NAME = 'watch_later'
1549 _PLAYLIST_TITLE = 'Youtube Watch Later'
1550 _PERSONAL_FEED = True
1551
1552
1553 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
1554 IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
1555 _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
1556 _FEED_NAME = 'history'
1557 _PERSONAL_FEED = True
1558 _PLAYLIST_TITLE = 'Youtube Watch History'
1559
1560
1561 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
1562 IE_NAME = 'youtube:favorites'
1563 IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
1564 _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
1565 _LOGIN_REQUIRED = True
1566
1567 def _real_extract(self, url):
1568 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
1569 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
1570 return self.url_result(playlist_id, 'YoutubePlaylist')
1571
1572
1573 class YoutubeSubscriptionsIE(YoutubePlaylistIE):
1574 IE_NAME = 'youtube:subscriptions'
1575 IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
1576 _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
1577 _TESTS = []
1578
1579 def _real_extract(self, url):
1580 title = 'Youtube Subscriptions'
1581 page = self._download_webpage('https://www.youtube.com/feed/subscriptions', title)
1582
1583 # The extraction process is the same as for playlists, but the regex
1584 # for the video ids doesn't contain an index
1585 ids = []
1586 more_widget_html = content_html = page
1587
1588 for page_num in itertools.count(1):
1589 matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
1590 new_ids = orderedSet(matches)
1591 ids.extend(new_ids)
1592
1593 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
1594 if not mobj:
1595 break
1596
1597 more = self._download_json(
1598 'https://youtube.com/%s' % mobj.group('more'), title,
1599 'Downloading page #%s' % page_num,
1600 transform_source=uppercase_escape)
1601 content_html = more['content_html']
1602 more_widget_html = more['load_more_widget_html']
1603
1604 return {
1605 '_type': 'playlist',
1606 'title': title,
1607 'entries': self._ids_to_results(ids),
1608 }
1609
1610
1611 class YoutubeTruncatedURLIE(InfoExtractor):
1612 IE_NAME = 'youtube:truncated_url'
1613 IE_DESC = False # Do not list
1614 _VALID_URL = r'''(?x)
1615 (?:https?://)?[^/]+/watch\?(?:
1616 feature=[a-z_]+|
1617 annotation_id=annotation_[^&]+
1618 )?$|
1619 (?:https?://)?(?:www\.)?youtube\.com/attribution_link\?a=[^&]+$
1620 '''
1621
1622 _TESTS = [{
1623 'url': 'http://www.youtube.com/watch?annotation_id=annotation_3951667041',
1624 'only_matching': True,
1625 }, {
1626 'url': 'http://www.youtube.com/watch?',
1627 'only_matching': True,
1628 }]
1629
1630 def _real_extract(self, url):
1631 raise ExtractorError(
1632 'Did you forget to quote the URL? Remember that & is a meta '
1633 'character in most shells, so you want to put the URL in quotes, '
1634 'like youtube-dl '
1635 '"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
1636 ' or simply youtube-dl BaW_jenozKc .',
1637 expected=True)