Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/youtube.py

   1 # coding: utf-8
   2
   3 import collections
   4 import errno
   5 import io
   6 import itertools
   7 import json
   8 import os.path
   9 import re
  10 import struct
  11 import traceback
  12 import zlib
  13
  14 from .common import InfoExtractor, SearchInfoExtractor
  15 from .subtitles import SubtitlesInfoExtractor
  16 from ..jsinterp import JSInterpreter
  17 from ..utils import (
  18     compat_chr,
  19     compat_parse_qs,
  20     compat_urllib_parse,
  21     compat_urllib_request,
  22     compat_urlparse,
  23     compat_str,
  24
  25     clean_html,
  26     get_cachedir,
  27     get_element_by_id,
  28     get_element_by_attribute,
  29     ExtractorError,
  30     int_or_none,
  31     PagedList,
  32     unescapeHTML,
  33     unified_strdate,
  34     orderedSet,
  35     write_json_file,
  36     uppercase_escape,
  37 )
  38
  39 class YoutubeBaseInfoExtractor(InfoExtractor):
  40     """Provide base functions for Youtube extractors"""
  41     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  42     _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
  43     _AGE_URL = 'https://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
  44     _NETRC_MACHINE = 'youtube'
  45     # If True it will raise an error if no login info is provided
  46     _LOGIN_REQUIRED = False
  47
  48     def _set_language(self):
  49         return bool(self._download_webpage(
  50             self._LANG_URL, None,
  51             note=u'Setting language', errnote='unable to set language',
  52             fatal=False))
  53
  54     def _login(self):
  55         (username, password) = self._get_login_info()
  56         # No authentication to be performed
  57         if username is None:
  58             if self._LOGIN_REQUIRED:
  59                 raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True)
  60             return False
  61
  62         login_page = self._download_webpage(
  63             self._LOGIN_URL, None,
  64             note=u'Downloading login page',
  65             errnote=u'unable to fetch login page', fatal=False)
  66         if login_page is False:
  67             return
  68
  69         galx = self._search_regex(r'(?s)<input.+?name="GALX".+?value="(.+?)"',
  70                                   login_page, u'Login GALX parameter')
  71
  72         # Log in
  73         login_form_strs = {
  74                 u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
  75                 u'Email': username,
  76                 u'GALX': galx,
  77                 u'Passwd': password,
  78                 u'PersistentCookie': u'yes',
  79                 u'_utf8': u'霱',
  80                 u'bgresponse': u'js_disabled',
  81                 u'checkConnection': u'',
  82                 u'checkedDomains': u'youtube',
  83                 u'dnConn': u'',
  84                 u'pstMsg': u'0',
  85                 u'rmShown': u'1',
  86                 u'secTok': u'',
  87                 u'signIn': u'Sign in',
  88                 u'timeStmp': u'',
  89                 u'service': u'youtube',
  90                 u'uilel': u'3',
  91                 u'hl': u'en_US',
  92         }
  93         # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
  94         # chokes on unicode
  95         login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
  96         login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
  97
  98         req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
  99         login_results = self._download_webpage(
 100             req, None,
 101             note=u'Logging in', errnote=u'unable to log in', fatal=False)
 102         if login_results is False:
 103             return False
 104         if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
 105             self._downloader.report_warning(u'unable to log in: bad username or password')
 106             return False
 107         return True
 108
 109     def _confirm_age(self):
 110         age_form = {
 111             'next_url': '/',
 112             'action_confirm': 'Confirm',
 113         }
 114         req = compat_urllib_request.Request(self._AGE_URL,
 115             compat_urllib_parse.urlencode(age_form).encode('ascii'))
 116
 117         self._download_webpage(
 118             req, None,
 119             note=u'Confirming age', errnote=u'Unable to confirm age')
 120         return True
 121
 122     def _real_initialize(self):
 123         if self._downloader is None:
 124             return
 125         if not self._set_language():
 126             return
 127         if not self._login():
 128             return
 129         self._confirm_age()
 130
 131
 132 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 133     IE_DESC = u'YouTube.com'
 134     _VALID_URL = r"""(?x)^
 135                      (
 136                          (?:https?://|//)?                                    # http(s):// or protocol-independent URL (optional)
 137                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
 138                             (?:www\.)?deturl\.com/www\.youtube\.com/|
 139                             (?:www\.)?pwnyoutube\.com/|
 140                             (?:www\.)?yourepeat\.com/|
 141                             tube\.majestyc\.net/|
 142                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
 143                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 144                          (?:                                                  # the various things that can precede the ID:
 145                              (?:(?:v|embed|e)/)                               # v/ or embed/ or e/
 146                              |(?:                                             # or the v= param in all its forms
 147                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 148                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 149                                  (?:.*?&)?                                    # any other preceding param (like /?s=tuff&v=xxxx)
 150                                  v=
 151                              )
 152                          ))
 153                          |youtu\.be/                                          # just youtu.be/xxxx
 154                          |https?://(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 155                          )
 156                      )?                                                       # all until now is optional -> you can pass the naked ID
 157                      ([0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
 158                      (?(1).+)?                                                # if we found the ID, everything can follow
 159                      $"""
 160     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
 161     _formats = {
 162         '5': {'ext': 'flv', 'width': 400, 'height': 240},
 163         '6': {'ext': 'flv', 'width': 450, 'height': 270},
 164         '13': {'ext': '3gp'},
 165         '17': {'ext': '3gp', 'width': 176, 'height': 144},
 166         '18': {'ext': 'mp4', 'width': 640, 'height': 360},
 167         '22': {'ext': 'mp4', 'width': 1280, 'height': 720},
 168         '34': {'ext': 'flv', 'width': 640, 'height': 360},
 169         '35': {'ext': 'flv', 'width': 854, 'height': 480},
 170         '36': {'ext': '3gp', 'width': 320, 'height': 240},
 171         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080},
 172         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072},
 173         '43': {'ext': 'webm', 'width': 640, 'height': 360},
 174         '44': {'ext': 'webm', 'width': 854, 'height': 480},
 175         '45': {'ext': 'webm', 'width': 1280, 'height': 720},
 176         '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
 177
 178
 179         # 3d videos
 180         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'preference': -20},
 181         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'preference': -20},
 182         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'preference': -20},
 183         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'preference': -20},
 184         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'preference': -20},
 185         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'preference': -20},
 186         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'preference': -20},
 187
 188         # Apple HTTP Live Streaming
 189         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
 190         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'preference': -10},
 191         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'preference': -10},
 192         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'preference': -10},
 193         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'preference': -10},
 194         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
 195         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'preference': -10},
 196
 197         # DASH mp4 video
 198         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 199         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 200         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 201         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 202         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 203         '138': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 204         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 205         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 206
 207         # Dash mp4 audio
 208         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
 209         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 128, 'preference': -50},
 210         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50},
 211
 212         # Dash webm
 213         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
 214         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
 215         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
 216         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
 217         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
 218         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
 219         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 220         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 221         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 222         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 223         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 224         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 225         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 226         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
 227
 228         # Dash webm audio
 229         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 48, 'preference': -50},
 230         '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 231
 232         # RTMP (unnamed)
 233         '_rtmp': {'protocol': 'rtmp'},
 234     }
 235
 236     IE_NAME = u'youtube'
 237     _TESTS = [
 238         {
 239             u"url":  u"http://www.youtube.com/watch?v=BaW_jenozKc",
 240             u"file":  u"BaW_jenozKc.mp4",
 241             u"info_dict": {
 242                 u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
 243                 u"uploader": u"Philipp Hagemeister",
 244                 u"uploader_id": u"phihag",
 245                 u"upload_date": u"20121002",
 246                 u"description": u"test chars:  \"'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .",
 247                 u"categories": [u'Science & Technology'],
 248             }
 249         },
 250         {
 251             u"url":  u"http://www.youtube.com/watch?v=UxxajLWwzqY",
 252             u"file":  u"UxxajLWwzqY.mp4",
 253             u"note": u"Test generic use_cipher_signature video (#897)",
 254             u"info_dict": {
 255                 u"upload_date": u"20120506",
 256                 u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
 257                 u"description": u"md5:fea86fda2d5a5784273df5c7cc994d9f",
 258                 u"uploader": u"Icona Pop",
 259                 u"uploader_id": u"IconaPop"
 260             }
 261         },
 262         {
 263             u"url":  u"https://www.youtube.com/watch?v=07FYdnEawAQ",
 264             u"file":  u"07FYdnEawAQ.mp4",
 265             u"note": u"Test VEVO video with age protection (#956)",
 266             u"info_dict": {
 267                 u"upload_date": u"20130703",
 268                 u"title": u"Justin Timberlake - Tunnel Vision (Explicit)",
 269                 u"description": u"md5:64249768eec3bc4276236606ea996373",
 270                 u"uploader": u"justintimberlakeVEVO",
 271                 u"uploader_id": u"justintimberlakeVEVO"
 272             }
 273         },
 274         {
 275             u"url":  u"//www.YouTube.com/watch?v=yZIXLfi8CZQ",
 276             u"file":  u"yZIXLfi8CZQ.mp4",
 277             u"note": u"Embed-only video (#1746)",
 278             u"info_dict": {
 279                 u"upload_date": u"20120608",
 280                 u"title": u"Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012",
 281                 u"description": u"md5:09b78bd971f1e3e289601dfba15ca4f7",
 282                 u"uploader": u"SET India",
 283                 u"uploader_id": u"setindia"
 284             }
 285         },
 286         {
 287             u"url": u"http://www.youtube.com/watch?v=a9LDPn-MO4I",
 288             u"file": u"a9LDPn-MO4I.m4a",
 289             u"note": u"256k DASH audio (format 141) via DASH manifest",
 290             u"info_dict": {
 291                 u"upload_date": "20121002",
 292                 u"uploader_id": "8KVIDEO",
 293                 u"description": "No description available.",
 294                 u"uploader": "8KVIDEO",
 295                 u"title": "UHDTV TEST 8K VIDEO.mp4"
 296             },
 297             u"params": {
 298                 u"youtube_include_dash_manifest": True,
 299                 u"format": "141",
 300             },
 301         },
 302         # DASH manifest with encrypted signature
 303         {
 304             u'url': u'https://www.youtube.com/watch?v=IB3lcPjvWLA',
 305             u'info_dict': {
 306                 u'id': u'IB3lcPjvWLA',
 307                 u'ext': u'm4a',
 308                 u'title': u'Afrojack - The Spark ft. Spree Wilson',
 309                 u'description': u'md5:9717375db5a9a3992be4668bbf3bc0a8',
 310                 u'uploader': u'AfrojackVEVO',
 311                 u'uploader_id': u'AfrojackVEVO',
 312                 u'upload_date': u'20131011',
 313             },
 314             u"params": {
 315                 u'youtube_include_dash_manifest': True,
 316                 u'format': '141',
 317             },
 318         },
 319     ]
 320
 321
 322     @classmethod
 323     def suitable(cls, url):
 324         """Receives a URL and returns True if suitable for this IE."""
 325         if YoutubePlaylistIE.suitable(url): return False
 326         return re.match(cls._VALID_URL, url) is not None
 327
 328     def __init__(self, *args, **kwargs):
 329         super(YoutubeIE, self).__init__(*args, **kwargs)
 330         self._player_cache = {}
 331
 332     def report_video_info_webpage_download(self, video_id):
 333         """Report attempt to download video info webpage."""
 334         self.to_screen(u'%s: Downloading video info webpage' % video_id)
 335
 336     def report_information_extraction(self, video_id):
 337         """Report attempt to extract video information."""
 338         self.to_screen(u'%s: Extracting video information' % video_id)
 339
 340     def report_unavailable_format(self, video_id, format):
 341         """Report extracted video URL."""
 342         self.to_screen(u'%s: Format %s not available' % (video_id, format))
 343
 344     def report_rtmp_download(self):
 345         """Indicate the download will use the RTMP protocol."""
 346         self.to_screen(u'RTMP download detected')
 347
 348     def _extract_signature_function(self, video_id, player_url, slen):
 349         id_m = re.match(r'.*-(?P<id>[a-zA-Z0-9_-]+)\.(?P<ext>[a-z]+)$',
 350                         player_url)
 351         player_type = id_m.group('ext')
 352         player_id = id_m.group('id')
 353
 354         # Read from filesystem cache
 355         func_id = '%s_%s_%d' % (player_type, player_id, slen)
 356         assert os.path.basename(func_id) == func_id
 357         cache_dir = get_cachedir(self._downloader.params)
 358
 359         cache_enabled = cache_dir is not None
 360         if cache_enabled:
 361             cache_fn = os.path.join(os.path.expanduser(cache_dir),
 362                                     u'youtube-sigfuncs',
 363                                     func_id + '.json')
 364             try:
 365                 with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
 366                     cache_spec = json.load(cachef)
 367                 return lambda s: u''.join(s[i] for i in cache_spec)
 368             except IOError:
 369                 pass  # No cache available
 370
 371         if player_type == 'js':
 372             code = self._download_webpage(
 373                 player_url, video_id,
 374                 note=u'Downloading %s player %s' % (player_type, player_id),
 375                 errnote=u'Download of %s failed' % player_url)
 376             res = self._parse_sig_js(code)
 377         elif player_type == 'swf':
 378             urlh = self._request_webpage(
 379                 player_url, video_id,
 380                 note=u'Downloading %s player %s' % (player_type, player_id),
 381                 errnote=u'Download of %s failed' % player_url)
 382             code = urlh.read()
 383             res = self._parse_sig_swf(code)
 384         else:
 385             assert False, 'Invalid player type %r' % player_type
 386
 387         if cache_enabled:
 388             try:
 389                 test_string = u''.join(map(compat_chr, range(slen)))
 390                 cache_res = res(test_string)
 391                 cache_spec = [ord(c) for c in cache_res]
 392                 try:
 393                     os.makedirs(os.path.dirname(cache_fn))
 394                 except OSError as ose:
 395                     if ose.errno != errno.EEXIST:
 396                         raise
 397                 write_json_file(cache_spec, cache_fn)
 398             except Exception:
 399                 tb = traceback.format_exc()
 400                 self._downloader.report_warning(
 401                     u'Writing cache to %r failed: %s' % (cache_fn, tb))
 402
 403         return res
 404
 405     def _print_sig_code(self, func, slen):
 406         def gen_sig_code(idxs):
 407             def _genslice(start, end, step):
 408                 starts = u'' if start == 0 else str(start)
 409                 ends = (u':%d' % (end+step)) if end + step >= 0 else u':'
 410                 steps = u'' if step == 1 else (u':%d' % step)
 411                 return u's[%s%s%s]' % (starts, ends, steps)
 412
 413             step = None
 414             start = '(Never used)'  # Quelch pyflakes warnings - start will be
 415                                     # set as soon as step is set
 416             for i, prev in zip(idxs[1:], idxs[:-1]):
 417                 if step is not None:
 418                     if i - prev == step:
 419                         continue
 420                     yield _genslice(start, prev, step)
 421                     step = None
 422                     continue
 423                 if i - prev in [-1, 1]:
 424                     step = i - prev
 425                     start = prev
 426                     continue
 427                 else:
 428                     yield u's[%d]' % prev
 429             if step is None:
 430                 yield u's[%d]' % i
 431             else:
 432                 yield _genslice(start, i, step)
 433
 434         test_string = u''.join(map(compat_chr, range(slen)))
 435         cache_res = func(test_string)
 436         cache_spec = [ord(c) for c in cache_res]
 437         expr_code = u' + '.join(gen_sig_code(cache_spec))
 438         code = u'if len(s) == %d:\n    return %s\n' % (slen, expr_code)
 439         self.to_screen(u'Extracted signature function:\n' + code)
 440
 441     def _parse_sig_js(self, jscode):
 442         funcname = self._search_regex(
 443             r'signature=([a-zA-Z]+)', jscode,
 444              u'Initial JS player signature function name')
 445
 446         jsi = JSInterpreter(jscode)
 447         initial_function = jsi.extract_function(funcname)
 448         return lambda s: initial_function([s])
 449
 450     def _parse_sig_swf(self, file_contents):
 451         if file_contents[1:3] != b'WS':
 452             raise ExtractorError(
 453                 u'Not an SWF file; header is %r' % file_contents[:3])
 454         if file_contents[:1] == b'C':
 455             content = zlib.decompress(file_contents[8:])
 456         else:
 457             raise NotImplementedError(u'Unsupported compression format %r' %
 458                                       file_contents[:1])
 459
 460         def extract_tags(content):
 461             pos = 0
 462             while pos < len(content):
 463                 header16 = struct.unpack('<H', content[pos:pos+2])[0]
 464                 pos += 2
 465                 tag_code = header16 >> 6
 466                 tag_len = header16 & 0x3f
 467                 if tag_len == 0x3f:
 468                     tag_len = struct.unpack('<I', content[pos:pos+4])[0]
 469                     pos += 4
 470                 assert pos+tag_len <= len(content)
 471                 yield (tag_code, content[pos:pos+tag_len])
 472                 pos += tag_len
 473
 474         code_tag = next(tag
 475                         for tag_code, tag in extract_tags(content)
 476                         if tag_code == 82)
 477         p = code_tag.index(b'\0', 4) + 1
 478         code_reader = io.BytesIO(code_tag[p:])
 479
 480         # Parse ABC (AVM2 ByteCode)
 481         def read_int(reader=None):
 482             if reader is None:
 483                 reader = code_reader
 484             res = 0
 485             shift = 0
 486             for _ in range(5):
 487                 buf = reader.read(1)
 488                 assert len(buf) == 1
 489                 b = struct.unpack('<B', buf)[0]
 490                 res = res | ((b & 0x7f) << shift)
 491                 if b & 0x80 == 0:
 492                     break
 493                 shift += 7
 494             return res
 495
 496         def u30(reader=None):
 497             res = read_int(reader)
 498             assert res & 0xf0000000 == 0
 499             return res
 500         u32 = read_int
 501
 502         def s32(reader=None):
 503             v = read_int(reader)
 504             if v & 0x80000000 != 0:
 505                 v = - ((v ^ 0xffffffff) + 1)
 506             return v
 507
 508         def read_string(reader=None):
 509             if reader is None:
 510                 reader = code_reader
 511             slen = u30(reader)
 512             resb = reader.read(slen)
 513             assert len(resb) == slen
 514             return resb.decode('utf-8')
 515
 516         def read_bytes(count, reader=None):
 517             if reader is None:
 518                 reader = code_reader
 519             resb = reader.read(count)
 520             assert len(resb) == count
 521             return resb
 522
 523         def read_byte(reader=None):
 524             resb = read_bytes(1, reader=reader)
 525             res = struct.unpack('<B', resb)[0]
 526             return res
 527
 528         # minor_version + major_version
 529         read_bytes(2 + 2)
 530
 531         # Constant pool
 532         int_count = u30()
 533         for _c in range(1, int_count):
 534             s32()
 535         uint_count = u30()
 536         for _c in range(1, uint_count):
 537             u32()
 538         double_count = u30()
 539         read_bytes((double_count-1) * 8)
 540         string_count = u30()
 541         constant_strings = [u'']
 542         for _c in range(1, string_count):
 543             s = read_string()
 544             constant_strings.append(s)
 545         namespace_count = u30()
 546         for _c in range(1, namespace_count):
 547             read_bytes(1)  # kind
 548             u30()  # name
 549         ns_set_count = u30()
 550         for _c in range(1, ns_set_count):
 551             count = u30()
 552             for _c2 in range(count):
 553                 u30()
 554         multiname_count = u30()
 555         MULTINAME_SIZES = {
 556             0x07: 2,  # QName
 557             0x0d: 2,  # QNameA
 558             0x0f: 1,  # RTQName
 559             0x10: 1,  # RTQNameA
 560             0x11: 0,  # RTQNameL
 561             0x12: 0,  # RTQNameLA
 562             0x09: 2,  # Multiname
 563             0x0e: 2,  # MultinameA
 564             0x1b: 1,  # MultinameL
 565             0x1c: 1,  # MultinameLA
 566         }
 567         multinames = [u'']
 568         for _c in range(1, multiname_count):
 569             kind = u30()
 570             assert kind in MULTINAME_SIZES, u'Invalid multiname kind %r' % kind
 571             if kind == 0x07:
 572                 u30()  # namespace_idx
 573                 name_idx = u30()
 574                 multinames.append(constant_strings[name_idx])
 575             else:
 576                 multinames.append('[MULTINAME kind: %d]' % kind)
 577                 for _c2 in range(MULTINAME_SIZES[kind]):
 578                     u30()
 579
 580         # Methods
 581         method_count = u30()
 582         MethodInfo = collections.namedtuple(
 583             'MethodInfo',
 584             ['NEED_ARGUMENTS', 'NEED_REST'])
 585         method_infos = []
 586         for method_id in range(method_count):
 587             param_count = u30()
 588             u30()  # return type
 589             for _ in range(param_count):
 590                 u30()  # param type
 591             u30()  # name index (always 0 for youtube)
 592             flags = read_byte()
 593             if flags & 0x08 != 0:
 594                 # Options present
 595                 option_count = u30()
 596                 for c in range(option_count):
 597                     u30()  # val
 598                     read_bytes(1)  # kind
 599             if flags & 0x80 != 0:
 600                 # Param names present
 601                 for _ in range(param_count):
 602                     u30()  # param name
 603             mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
 604             method_infos.append(mi)
 605
 606         # Metadata
 607         metadata_count = u30()
 608         for _c in range(metadata_count):
 609             u30()  # name
 610             item_count = u30()
 611             for _c2 in range(item_count):
 612                 u30()  # key
 613                 u30()  # value
 614
 615         def parse_traits_info():
 616             trait_name_idx = u30()
 617             kind_full = read_byte()
 618             kind = kind_full & 0x0f
 619             attrs = kind_full >> 4
 620             methods = {}
 621             if kind in [0x00, 0x06]:  # Slot or Const
 622                 u30()  # Slot id
 623                 u30()  # type_name_idx
 624                 vindex = u30()
 625                 if vindex != 0:
 626                     read_byte()  # vkind
 627             elif kind in [0x01, 0x02, 0x03]:  # Method / Getter / Setter
 628                 u30()  # disp_id
 629                 method_idx = u30()
 630                 methods[multinames[trait_name_idx]] = method_idx
 631             elif kind == 0x04:  # Class
 632                 u30()  # slot_id
 633                 u30()  # classi
 634             elif kind == 0x05:  # Function
 635                 u30()  # slot_id
 636                 function_idx = u30()
 637                 methods[function_idx] = multinames[trait_name_idx]
 638             else:
 639                 raise ExtractorError(u'Unsupported trait kind %d' % kind)
 640
 641             if attrs & 0x4 != 0:  # Metadata present
 642                 metadata_count = u30()
 643                 for _c3 in range(metadata_count):
 644                     u30()  # metadata index
 645
 646             return methods
 647
 648         # Classes
 649         TARGET_CLASSNAME = u'SignatureDecipher'
 650         searched_idx = multinames.index(TARGET_CLASSNAME)
 651         searched_class_id = None
 652         class_count = u30()
 653         for class_id in range(class_count):
 654             name_idx = u30()
 655             if name_idx == searched_idx:
 656                 # We found the class we're looking for!
 657                 searched_class_id = class_id
 658             u30()  # super_name idx
 659             flags = read_byte()
 660             if flags & 0x08 != 0:  # Protected namespace is present
 661                 u30()  # protected_ns_idx
 662             intrf_count = u30()
 663             for _c2 in range(intrf_count):
 664                 u30()
 665             u30()  # iinit
 666             trait_count = u30()
 667             for _c2 in range(trait_count):
 668                 parse_traits_info()
 669
 670         if searched_class_id is None:
 671             raise ExtractorError(u'Target class %r not found' %
 672                                  TARGET_CLASSNAME)
 673
 674         method_names = {}
 675         method_idxs = {}
 676         for class_id in range(class_count):
 677             u30()  # cinit
 678             trait_count = u30()
 679             for _c2 in range(trait_count):
 680                 trait_methods = parse_traits_info()
 681                 if class_id == searched_class_id:
 682                     method_names.update(trait_methods.items())
 683                     method_idxs.update(dict(
 684                         (idx, name)
 685                         for name, idx in trait_methods.items()))
 686
 687         # Scripts
 688         script_count = u30()
 689         for _c in range(script_count):
 690             u30()  # init
 691             trait_count = u30()
 692             for _c2 in range(trait_count):
 693                 parse_traits_info()
 694
 695         # Method bodies
 696         method_body_count = u30()
 697         Method = collections.namedtuple('Method', ['code', 'local_count'])
 698         methods = {}
 699         for _c in range(method_body_count):
 700             method_idx = u30()
 701             u30()  # max_stack
 702             local_count = u30()
 703             u30()  # init_scope_depth
 704             u30()  # max_scope_depth
 705             code_length = u30()
 706             code = read_bytes(code_length)
 707             if method_idx in method_idxs:
 708                 m = Method(code, local_count)
 709                 methods[method_idxs[method_idx]] = m
 710             exception_count = u30()
 711             for _c2 in range(exception_count):
 712                 u30()  # from
 713                 u30()  # to
 714                 u30()  # target
 715                 u30()  # exc_type
 716                 u30()  # var_name
 717             trait_count = u30()
 718             for _c2 in range(trait_count):
 719                 parse_traits_info()
 720
 721         assert p + code_reader.tell() == len(code_tag)
 722         assert len(methods) == len(method_idxs)
 723
 724         method_pyfunctions = {}
 725
 726         def extract_function(func_name):
 727             if func_name in method_pyfunctions:
 728                 return method_pyfunctions[func_name]
 729             if func_name not in methods:
 730                 raise ExtractorError(u'Cannot find function %r' % func_name)
 731             m = methods[func_name]
 732
 733             def resfunc(args):
 734                 registers = ['(this)'] + list(args) + [None] * m.local_count
 735                 stack = []
 736                 coder = io.BytesIO(m.code)
 737                 while True:
 738                     opcode = struct.unpack('!B', coder.read(1))[0]
 739                     if opcode == 36:  # pushbyte
 740                         v = struct.unpack('!B', coder.read(1))[0]
 741                         stack.append(v)
 742                     elif opcode == 44:  # pushstring
 743                         idx = u30(coder)
 744                         stack.append(constant_strings[idx])
 745                     elif opcode == 48:  # pushscope
 746                         # We don't implement the scope register, so we'll just
 747                         # ignore the popped value
 748                         stack.pop()
 749                     elif opcode == 70:  # callproperty
 750                         index = u30(coder)
 751                         mname = multinames[index]
 752                         arg_count = u30(coder)
 753                         args = list(reversed(
 754                             [stack.pop() for _ in range(arg_count)]))
 755                         obj = stack.pop()
 756                         if mname == u'split':
 757                             assert len(args) == 1
 758                             assert isinstance(args[0], compat_str)
 759                             assert isinstance(obj, compat_str)
 760                             if args[0] == u'':
 761                                 res = list(obj)
 762                             else:
 763                                 res = obj.split(args[0])
 764                             stack.append(res)
 765                         elif mname == u'slice':
 766                             assert len(args) == 1
 767                             assert isinstance(args[0], int)
 768                             assert isinstance(obj, list)
 769                             res = obj[args[0]:]
 770                             stack.append(res)
 771                         elif mname == u'join':
 772                             assert len(args) == 1
 773                             assert isinstance(args[0], compat_str)
 774                             assert isinstance(obj, list)
 775                             res = args[0].join(obj)
 776                             stack.append(res)
 777                         elif mname in method_pyfunctions:
 778                             stack.append(method_pyfunctions[mname](args))
 779                         else:
 780                             raise NotImplementedError(
 781                                 u'Unsupported property %r on %r'
 782                                 % (mname, obj))
 783                     elif opcode == 72:  # returnvalue
 784                         res = stack.pop()
 785                         return res
 786                     elif opcode == 79:  # callpropvoid
 787                         index = u30(coder)
 788                         mname = multinames[index]
 789                         arg_count = u30(coder)
 790                         args = list(reversed(
 791                             [stack.pop() for _ in range(arg_count)]))
 792                         obj = stack.pop()
 793                         if mname == u'reverse':
 794                             assert isinstance(obj, list)
 795                             obj.reverse()
 796                         else:
 797                             raise NotImplementedError(
 798                                 u'Unsupported (void) property %r on %r'
 799                                 % (mname, obj))
 800                     elif opcode == 93:  # findpropstrict
 801                         index = u30(coder)
 802                         mname = multinames[index]
 803                         res = extract_function(mname)
 804                         stack.append(res)
 805                     elif opcode == 97:  # setproperty
 806                         index = u30(coder)
 807                         value = stack.pop()
 808                         idx = stack.pop()
 809                         obj = stack.pop()
 810                         assert isinstance(obj, list)
 811                         assert isinstance(idx, int)
 812                         obj[idx] = value
 813                     elif opcode == 98:  # getlocal
 814                         index = u30(coder)
 815                         stack.append(registers[index])
 816                     elif opcode == 99:  # setlocal
 817                         index = u30(coder)
 818                         value = stack.pop()
 819                         registers[index] = value
 820                     elif opcode == 102:  # getproperty
 821                         index = u30(coder)
 822                         pname = multinames[index]
 823                         if pname == u'length':
 824                             obj = stack.pop()
 825                             assert isinstance(obj, list)
 826                             stack.append(len(obj))
 827                         else:  # Assume attribute access
 828                             idx = stack.pop()
 829                             assert isinstance(idx, int)
 830                             obj = stack.pop()
 831                             assert isinstance(obj, list)
 832                             stack.append(obj[idx])
 833                     elif opcode == 128:  # coerce
 834                         u30(coder)
 835                     elif opcode == 133:  # coerce_s
 836                         assert isinstance(stack[-1], (type(None), compat_str))
 837                     elif opcode == 164:  # modulo
 838                         value2 = stack.pop()
 839                         value1 = stack.pop()
 840                         res = value1 % value2
 841                         stack.append(res)
 842                     elif opcode == 208:  # getlocal_0
 843                         stack.append(registers[0])
 844                     elif opcode == 209:  # getlocal_1
 845                         stack.append(registers[1])
 846                     elif opcode == 210:  # getlocal_2
 847                         stack.append(registers[2])
 848                     elif opcode == 211:  # getlocal_3
 849                         stack.append(registers[3])
 850                     elif opcode == 214:  # setlocal_2
 851                         registers[2] = stack.pop()
 852                     elif opcode == 215:  # setlocal_3
 853                         registers[3] = stack.pop()
 854                     else:
 855                         raise NotImplementedError(
 856                             u'Unsupported opcode %d' % opcode)
 857
 858             method_pyfunctions[func_name] = resfunc
 859             return resfunc
 860
 861         initial_function = extract_function(u'decipher')
 862         return lambda s: initial_function([s])
 863
 864     def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
 865         """Turn the encrypted s field into a working signature"""
 866
 867         if player_url is not None:
 868             if player_url.startswith(u'//'):
 869                 player_url = u'https:' + player_url
 870             try:
 871                 player_id = (player_url, len(s))
 872                 if player_id not in self._player_cache:
 873                     func = self._extract_signature_function(
 874                         video_id, player_url, len(s)
 875                     )
 876                     self._player_cache[player_id] = func
 877                 func = self._player_cache[player_id]
 878                 if self._downloader.params.get('youtube_print_sig_code'):
 879                     self._print_sig_code(func, len(s))
 880                 return func(s)
 881             except Exception:
 882                 tb = traceback.format_exc()
 883                 self._downloader.report_warning(
 884                     u'Automatic signature extraction failed: ' + tb)
 885
 886             self._downloader.report_warning(
 887                 u'Warning: Falling back to static signature algorithm')
 888
 889         return self._static_decrypt_signature(
 890             s, video_id, player_url, age_gate)
 891
 892     def _static_decrypt_signature(self, s, video_id, player_url, age_gate):
 893         if age_gate:
 894             # The videos with age protection use another player, so the
 895             # algorithms can be different.
 896             if len(s) == 86:
 897                 return s[2:63] + s[82] + s[64:82] + s[63]
 898
 899         if len(s) == 93:
 900             return s[86:29:-1] + s[88] + s[28:5:-1]
 901         elif len(s) == 92:
 902             return s[25] + s[3:25] + s[0] + s[26:42] + s[79] + s[43:79] + s[91] + s[80:83]
 903         elif len(s) == 91:
 904             return s[84:27:-1] + s[86] + s[26:5:-1]
 905         elif len(s) == 90:
 906             return s[25] + s[3:25] + s[2] + s[26:40] + s[77] + s[41:77] + s[89] + s[78:81]
 907         elif len(s) == 89:
 908             return s[84:78:-1] + s[87] + s[77:60:-1] + s[0] + s[59:3:-1]
 909         elif len(s) == 88:
 910             return s[7:28] + s[87] + s[29:45] + s[55] + s[46:55] + s[2] + s[56:87] + s[28]
 911         elif len(s) == 87:
 912             return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:]
 913         elif len(s) == 86:
 914             return s[80:72:-1] + s[16] + s[71:39:-1] + s[72] + s[38:16:-1] + s[82] + s[15::-1]
 915         elif len(s) == 85:
 916             return s[3:11] + s[0] + s[12:55] + s[84] + s[56:84]
 917         elif len(s) == 84:
 918             return s[78:70:-1] + s[14] + s[69:37:-1] + s[70] + s[36:14:-1] + s[80] + s[:14][::-1]
 919         elif len(s) == 83:
 920             return s[80:63:-1] + s[0] + s[62:0:-1] + s[63]
 921         elif len(s) == 82:
 922             return s[80:37:-1] + s[7] + s[36:7:-1] + s[0] + s[6:0:-1] + s[37]
 923         elif len(s) == 81:
 924             return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
 925         elif len(s) == 80:
 926             return s[1:19] + s[0] + s[20:68] + s[19] + s[69:80]
 927         elif len(s) == 79:
 928             return s[54] + s[77:54:-1] + s[39] + s[53:39:-1] + s[78] + s[38:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
 929
 930         else:
 931             raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
 932
 933     def _get_available_subtitles(self, video_id, webpage):
 934         try:
 935             sub_list = self._download_webpage(
 936                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
 937                 video_id, note=False)
 938         except ExtractorError as err:
 939             self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
 940             return {}
 941         lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
 942
 943         sub_lang_list = {}
 944         for l in lang_list:
 945             lang = l[1]
 946             params = compat_urllib_parse.urlencode({
 947                 'lang': lang,
 948                 'v': video_id,
 949                 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
 950                 'name': unescapeHTML(l[0]).encode('utf-8'),
 951             })
 952             url = u'https://www.youtube.com/api/timedtext?' + params
 953             sub_lang_list[lang] = url
 954         if not sub_lang_list:
 955             self._downloader.report_warning(u'video doesn\'t have subtitles')
 956             return {}
 957         return sub_lang_list
 958
 959     def _get_available_automatic_caption(self, video_id, webpage):
 960         """We need the webpage for getting the captions url, pass it as an
 961            argument to speed up the process."""
 962         sub_format = self._downloader.params.get('subtitlesformat', 'srt')
 963         self.to_screen(u'%s: Looking for automatic captions' % video_id)
 964         mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
 965         err_msg = u'Couldn\'t find automatic captions for %s' % video_id
 966         if mobj is None:
 967             self._downloader.report_warning(err_msg)
 968             return {}
 969         player_config = json.loads(mobj.group(1))
 970         try:
 971             args = player_config[u'args']
 972             caption_url = args[u'ttsurl']
 973             timestamp = args[u'timestamp']
 974             # We get the available subtitles
 975             list_params = compat_urllib_parse.urlencode({
 976                 'type': 'list',
 977                 'tlangs': 1,
 978                 'asrs': 1,
 979             })
 980             list_url = caption_url + '&' + list_params
 981             caption_list = self._download_xml(list_url, video_id)
 982             original_lang_node = caption_list.find('track')
 983             if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' :
 984                 self._downloader.report_warning(u'Video doesn\'t have automatic captions')
 985                 return {}
 986             original_lang = original_lang_node.attrib['lang_code']
 987
 988             sub_lang_list = {}
 989             for lang_node in caption_list.findall('target'):
 990                 sub_lang = lang_node.attrib['lang_code']
 991                 params = compat_urllib_parse.urlencode({
 992                     'lang': original_lang,
 993                     'tlang': sub_lang,
 994                     'fmt': sub_format,
 995                     'ts': timestamp,
 996                     'kind': 'asr',
 997                 })
 998                 sub_lang_list[sub_lang] = caption_url + '&' + params
 999             return sub_lang_list
1000         # An extractor error can be raise by the download process if there are
1001         # no automatic captions but there are subtitles
1002         except (KeyError, ExtractorError):
1003             self._downloader.report_warning(err_msg)
1004             return {}
1005
1006     @classmethod
1007     def extract_id(cls, url):
1008         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1009         if mobj is None:
1010             raise ExtractorError(u'Invalid URL: %s' % url)
1011         video_id = mobj.group(2)
1012         return video_id
1013
1014     def _extract_from_m3u8(self, manifest_url, video_id):
1015         url_map = {}
1016         def _get_urls(_manifest):
1017             lines = _manifest.split('\n')
1018             urls = filter(lambda l: l and not l.startswith('#'),
1019                             lines)
1020             return urls
1021         manifest = self._download_webpage(manifest_url, video_id, u'Downloading formats manifest')
1022         formats_urls = _get_urls(manifest)
1023         for format_url in formats_urls:
1024             itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
1025             url_map[itag] = format_url
1026         return url_map
1027
1028     def _extract_annotations(self, video_id):
1029         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1030         return self._download_webpage(url, video_id, note=u'Searching for annotations.', errnote=u'Unable to download video annotations.')
1031
1032     def _real_extract(self, url):
1033         proto = (
1034             u'http' if self._downloader.params.get('prefer_insecure', False)
1035             else u'https')
1036
1037         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1038         mobj = re.search(self._NEXT_URL_RE, url)
1039         if mobj:
1040             url = proto + '://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
1041         video_id = self.extract_id(url)
1042
1043         # Get video webpage
1044         url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
1045         video_webpage = self._download_webpage(url, video_id)
1046
1047         # Attempt to extract SWF player URL
1048         mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1049         if mobj is not None:
1050             player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1051         else:
1052             player_url = None
1053
1054         # Get video info
1055         self.report_video_info_webpage_download(video_id)
1056         if re.search(r'player-age-gate-content">', video_webpage) is not None:
1057             self.report_age_confirmation()
1058             age_gate = True
1059             # We simulate the access to the video from www.youtube.com/v/{video_id}
1060             # this can be viewed without login into Youtube
1061             data = compat_urllib_parse.urlencode({'video_id': video_id,
1062                                                   'el': 'player_embedded',
1063                                                   'gl': 'US',
1064                                                   'hl': 'en',
1065                                                   'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1066                                                   'asv': 3,
1067                                                   'sts':'1588',
1068                                                   })
1069             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1070             video_info_webpage = self._download_webpage(video_info_url, video_id,
1071                                     note=False,
1072                                     errnote='unable to download video info webpage')
1073             video_info = compat_parse_qs(video_info_webpage)
1074         else:
1075             age_gate = False
1076             for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1077                 video_info_url = (proto + '://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1078                         % (video_id, el_type))
1079                 video_info_webpage = self._download_webpage(video_info_url, video_id,
1080                                         note=False,
1081                                         errnote='unable to download video info webpage')
1082                 video_info = compat_parse_qs(video_info_webpage)
1083                 if 'token' in video_info:
1084                     break
1085         if 'token' not in video_info:
1086             if 'reason' in video_info:
1087                 raise ExtractorError(
1088                     u'YouTube said: %s' % video_info['reason'][0],
1089                     expected=True, video_id=video_id)
1090             else:
1091                 raise ExtractorError(
1092                     u'"token" parameter not in video info for unknown reason',
1093                     video_id=video_id)
1094
1095         if 'view_count' in video_info:
1096             view_count = int(video_info['view_count'][0])
1097         else:
1098             view_count = None
1099
1100         # Check for "rental" videos
1101         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1102             raise ExtractorError(u'"rental" videos not supported')
1103
1104         # Start extracting information
1105         self.report_information_extraction(video_id)
1106
1107         # uploader
1108         if 'author' not in video_info:
1109             raise ExtractorError(u'Unable to extract uploader name')
1110         video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
1111
1112         # uploader_id
1113         video_uploader_id = None
1114         mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
1115         if mobj is not None:
1116             video_uploader_id = mobj.group(1)
1117         else:
1118             self._downloader.report_warning(u'unable to extract uploader nickname')
1119
1120         # title
1121         if 'title' in video_info:
1122             video_title = video_info['title'][0]
1123         else:
1124             self._downloader.report_warning(u'Unable to extract video title')
1125             video_title = u'_'
1126
1127         # thumbnail image
1128         # We try first to get a high quality image:
1129         m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1130                             video_webpage, re.DOTALL)
1131         if m_thumb is not None:
1132             video_thumbnail = m_thumb.group(1)
1133         elif 'thumbnail_url' not in video_info:
1134             self._downloader.report_warning(u'unable to extract video thumbnail')
1135             video_thumbnail = None
1136         else:   # don't panic if we can't find it
1137             video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
1138
1139         # upload date
1140         upload_date = None
1141         mobj = re.search(r'(?s)id="eow-date.*?>(.*?)</span>', video_webpage)
1142         if mobj is None:
1143             mobj = re.search(
1144                 r'(?s)id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live) on (.*?)</strong>',
1145                 video_webpage)
1146         if mobj is not None:
1147             upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1148             upload_date = unified_strdate(upload_date)
1149
1150         m_cat_container = get_element_by_id("eow-category", video_webpage)
1151         if m_cat_container:
1152             category = self._html_search_regex(
1153                 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
1154                 default=None)
1155             video_categories = None if category is None else [category]
1156         else:
1157             video_categories = None
1158
1159         # description
1160         video_description = get_element_by_id("eow-description", video_webpage)
1161         if video_description:
1162             video_description = re.sub(r'''(?x)
1163                 <a\s+
1164                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
1165                     title="([^"]+)"\s+
1166                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
1167                     class="yt-uix-redirect-link"\s*>
1168                 [^<]+
1169                 </a>
1170             ''', r'\1', video_description)
1171             video_description = clean_html(video_description)
1172         else:
1173             fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1174             if fd_mobj:
1175                 video_description = unescapeHTML(fd_mobj.group(1))
1176             else:
1177                 video_description = u''
1178
1179         def _extract_count(klass):
1180             count = self._search_regex(
1181                 r'class="%s">([\d,]+)</span>' % re.escape(klass),
1182                 video_webpage, klass, default=None)
1183             if count is not None:
1184                 return int(count.replace(',', ''))
1185             return None
1186         like_count = _extract_count(u'likes-count')
1187         dislike_count = _extract_count(u'dislikes-count')
1188
1189         # subtitles
1190         video_subtitles = self.extract_subtitles(video_id, video_webpage)
1191
1192         if self._downloader.params.get('listsubtitles', False):
1193             self._list_available_subtitles(video_id, video_webpage)
1194             return
1195
1196         if 'length_seconds' not in video_info:
1197             self._downloader.report_warning(u'unable to extract video duration')
1198             video_duration = None
1199         else:
1200             video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]))
1201
1202         # annotations
1203         video_annotations = None
1204         if self._downloader.params.get('writeannotations', False):
1205                 video_annotations = self._extract_annotations(video_id)
1206
1207         # Decide which formats to download
1208         try:
1209             mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
1210             if not mobj:
1211                 raise ValueError('Could not find vevo ID')
1212             json_code = uppercase_escape(mobj.group(1))
1213             ytplayer_config = json.loads(json_code)
1214             args = ytplayer_config['args']
1215             # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
1216             # this signatures are encrypted
1217             if 'url_encoded_fmt_stream_map' not in args:
1218                 raise ValueError(u'No stream_map present')  # caught below
1219             re_signature = re.compile(r'[&,]s=')
1220             m_s = re_signature.search(args['url_encoded_fmt_stream_map'])
1221             if m_s is not None:
1222                 self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
1223                 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
1224             m_s = re_signature.search(args.get('adaptive_fmts', u''))
1225             if m_s is not None:
1226                 if 'adaptive_fmts' in video_info:
1227                     video_info['adaptive_fmts'][0] += ',' + args['adaptive_fmts']
1228                 else:
1229                     video_info['adaptive_fmts'] = [args['adaptive_fmts']]
1230         except ValueError:
1231             pass
1232
1233         def _map_to_format_list(urlmap):
1234             formats = []
1235             for itag, video_real_url in urlmap.items():
1236                 dct = {
1237                     'format_id': itag,
1238                     'url': video_real_url,
1239                     'player_url': player_url,
1240                 }
1241                 if itag in self._formats:
1242                     dct.update(self._formats[itag])
1243                 formats.append(dct)
1244             return formats
1245
1246         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1247             self.report_rtmp_download()
1248             formats = [{
1249                 'format_id': '_rtmp',
1250                 'protocol': 'rtmp',
1251                 'url': video_info['conn'][0],
1252                 'player_url': player_url,
1253             }]
1254         elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
1255             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts',[''])[0]
1256             if 'rtmpe%3Dyes' in encoded_url_map:
1257                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1258             url_map = {}
1259             for url_data_str in encoded_url_map.split(','):
1260                 url_data = compat_parse_qs(url_data_str)
1261                 if 'itag' in url_data and 'url' in url_data:
1262                     url = url_data['url'][0]
1263                     if 'sig' in url_data:
1264                         url += '&signature=' + url_data['sig'][0]
1265                     elif 's' in url_data:
1266                         encrypted_sig = url_data['s'][0]
1267                         if self._downloader.params.get('verbose'):
1268                             if age_gate:
1269                                 if player_url is None:
1270                                     player_version = 'unknown'
1271                                 else:
1272                                     player_version = self._search_regex(
1273                                         r'-(.+)\.swf$', player_url,
1274                                         u'flash player', fatal=False)
1275                                 player_desc = 'flash player %s' % player_version
1276                             else:
1277                                 player_version = self._search_regex(
1278                                     r'html5player-(.+?)\.js', video_webpage,
1279                                     'html5 player', fatal=False)
1280                                 player_desc = u'html5 player %s' % player_version
1281
1282                             parts_sizes = u'.'.join(compat_str(len(part)) for part in encrypted_sig.split('.'))
1283                             self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' %
1284                                 (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc))
1285
1286                         if not age_gate:
1287                             jsplayer_url_json = self._search_regex(
1288                                 r'"assets":.+?"js":\s*("[^"]+")',
1289                                 video_webpage, u'JS player URL')
1290                             player_url = json.loads(jsplayer_url_json)
1291
1292                         signature = self._decrypt_signature(
1293                             encrypted_sig, video_id, player_url, age_gate)
1294                         url += '&signature=' + signature
1295                     if 'ratebypass' not in url:
1296                         url += '&ratebypass=yes'
1297                     url_map[url_data['itag'][0]] = url
1298             formats = _map_to_format_list(url_map)
1299         elif video_info.get('hlsvp'):
1300             manifest_url = video_info['hlsvp'][0]
1301             url_map = self._extract_from_m3u8(manifest_url, video_id)
1302             formats = _map_to_format_list(url_map)
1303         else:
1304             raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1305
1306         # Look for the DASH manifest
1307         if (self._downloader.params.get('youtube_include_dash_manifest', False)):
1308             try:
1309                 # The DASH manifest used needs to be the one from the original video_webpage.
1310                 # The one found in get_video_info seems to be using different signatures.
1311                 # However, in the case of an age restriction there won't be any embedded dashmpd in the video_webpage.
1312                 # Luckily, it seems, this case uses some kind of default signature (len == 86), so the
1313                 # combination of get_video_info and the _static_decrypt_signature() decryption fallback will work here.
1314                 if age_gate:
1315                     dash_manifest_url = video_info.get('dashmpd')[0]
1316                 else:
1317                     dash_manifest_url = ytplayer_config['args']['dashmpd']
1318                 def decrypt_sig(mobj):
1319                     s = mobj.group(1)
1320                     dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
1321                     return '/signature/%s' % dec_s
1322                 dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
1323                 dash_doc = self._download_xml(
1324                     dash_manifest_url, video_id,
1325                     note=u'Downloading DASH manifest',
1326                     errnote=u'Could not download DASH manifest')
1327                 for r in dash_doc.findall(u'.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
1328                     url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
1329                     if url_el is None:
1330                         continue
1331                     format_id = r.attrib['id']
1332                     video_url = url_el.text
1333                     filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
1334                     f = {
1335                         'format_id': format_id,
1336                         'url': video_url,
1337                         'width': int_or_none(r.attrib.get('width')),
1338                         'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
1339                         'asr': int_or_none(r.attrib.get('audioSamplingRate')),
1340                         'filesize': filesize,
1341                     }
1342                     try:
1343                         existing_format = next(
1344                             fo for fo in formats
1345                             if fo['format_id'] == format_id)
1346                     except StopIteration:
1347                         f.update(self._formats.get(format_id, {}))
1348                         formats.append(f)
1349                     else:
1350                         existing_format.update(f)
1351
1352             except (ExtractorError, KeyError) as e:
1353                 self.report_warning(u'Skipping DASH manifest: %s' % e, video_id)
1354
1355         self._sort_formats(formats)
1356
1357         return {
1358             'id':           video_id,
1359             'uploader':     video_uploader,
1360             'uploader_id':  video_uploader_id,
1361             'upload_date':  upload_date,
1362             'title':        video_title,
1363             'thumbnail':    video_thumbnail,
1364             'description':  video_description,
1365             'categories':   video_categories,
1366             'subtitles':    video_subtitles,
1367             'duration':     video_duration,
1368             'age_limit':    18 if age_gate else 0,
1369             'annotations':  video_annotations,
1370             'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
1371             'view_count':   view_count,
1372             'like_count': like_count,
1373             'dislike_count': dislike_count,
1374             'formats':      formats,
1375         }
1376
1377 class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
1378     IE_DESC = u'YouTube.com playlists'
1379     _VALID_URL = r"""(?x)(?:
1380                         (?:https?://)?
1381                         (?:\w+\.)?
1382                         youtube\.com/
1383                         (?:
1384                            (?:course|view_play_list|my_playlists|artist|playlist|watch)
1385                            \? (?:.*?&)*? (?:p|a|list)=
1386                         |  p/
1387                         )
1388                         (
1389                             (?:PL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,}
1390                             # Top tracks, they can also include dots
1391                             |(?:MC)[\w\.]*
1392                         )
1393                         .*
1394                      |
1395                         ((?:PL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
1396                      )"""
1397     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
1398     _MORE_PAGES_INDICATOR = r'data-link-type="next"'
1399     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
1400     IE_NAME = u'youtube:playlist'
1401
1402     def _real_initialize(self):
1403         self._login()
1404
1405     def _ids_to_results(self, ids):
1406         return [self.url_result(vid_id, 'Youtube', video_id=vid_id)
1407                        for vid_id in ids]
1408
1409     def _extract_mix(self, playlist_id):
1410         # The mixes are generated from a a single video
1411         # the id of the playlist is just 'RD' + video_id
1412         url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
1413         webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix')
1414         search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
1415         title_span = (search_title('playlist-title') or
1416             search_title('title long-title') or search_title('title'))
1417         title = clean_html(title_span)
1418         video_re = r'''(?x)data-video-username=".*?".*?
1419                        href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id)
1420         ids = orderedSet(re.findall(video_re, webpage, flags=re.DOTALL))
1421         url_results = self._ids_to_results(ids)
1422
1423         return self.playlist_result(url_results, playlist_id, title)
1424
1425     def _real_extract(self, url):
1426         # Extract playlist id
1427         mobj = re.match(self._VALID_URL, url)
1428         if mobj is None:
1429             raise ExtractorError(u'Invalid URL: %s' % url)
1430         playlist_id = mobj.group(1) or mobj.group(2)
1431
1432         # Check if it's a video-specific URL
1433         query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
1434         if 'v' in query_dict:
1435             video_id = query_dict['v'][0]
1436             if self._downloader.params.get('noplaylist'):
1437                 self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id)
1438                 return self.url_result(video_id, 'Youtube', video_id=video_id)
1439             else:
1440                 self.to_screen(u'Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
1441
1442         if playlist_id.startswith('RD'):
1443             # Mixes require a custom extraction process
1444             return self._extract_mix(playlist_id)
1445         if playlist_id.startswith('TL'):
1446             raise ExtractorError(u'For downloading YouTube.com top lists, use '
1447                 u'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
1448
1449         url = self._TEMPLATE_URL % playlist_id
1450         page = self._download_webpage(url, playlist_id)
1451         more_widget_html = content_html = page
1452
1453         # Check if the playlist exists or is private
1454         if re.search(r'<div class="yt-alert-message">[^<]*?(The|This) playlist (does not exist|is private)[^<]*?</div>', page) is not None:
1455             raise ExtractorError(
1456                 u'The playlist doesn\'t exist or is private, use --username or '
1457                 '--netrc to access it.',
1458                 expected=True)
1459
1460         # Extract the video ids from the playlist pages
1461         ids = []
1462
1463         for page_num in itertools.count(1):
1464             matches = re.finditer(self._VIDEO_RE, content_html)
1465             # We remove the duplicates and the link with index 0
1466             # (it's not the first video of the playlist)
1467             new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
1468             ids.extend(new_ids)
1469
1470             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
1471             if not mobj:
1472                 break
1473
1474             more = self._download_json(
1475                 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
1476                 'Downloading page #%s' % page_num,
1477                 transform_source=uppercase_escape)
1478             content_html = more['content_html']
1479             more_widget_html = more['load_more_widget_html']
1480
1481         playlist_title = self._html_search_regex(
1482             r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
1483             page, u'title')
1484
1485         url_results = self._ids_to_results(ids)
1486         return self.playlist_result(url_results, playlist_id, playlist_title)
1487
1488
1489 class YoutubeTopListIE(YoutubePlaylistIE):
1490     IE_NAME = u'youtube:toplist'
1491     IE_DESC = (u'YouTube.com top lists, "yttoplist:{channel}:{list title}"'
1492         u' (Example: "yttoplist:music:Top Tracks")')
1493     _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
1494
1495     def _real_extract(self, url):
1496         mobj = re.match(self._VALID_URL, url)
1497         channel = mobj.group('chann')
1498         title = mobj.group('title')
1499         query = compat_urllib_parse.urlencode({'title': title})
1500         playlist_re = 'href="([^"]+?%s.*?)"' % re.escape(query)
1501         channel_page = self._download_webpage('https://www.youtube.com/%s' % channel, title)
1502         link = self._html_search_regex(playlist_re, channel_page, u'list')
1503         url = compat_urlparse.urljoin('https://www.youtube.com/', link)
1504
1505         video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
1506         ids = []
1507         # sometimes the webpage doesn't contain the videos
1508         # retry until we get them
1509         for i in itertools.count(0):
1510             msg = u'Downloading Youtube mix'
1511             if i > 0:
1512                 msg += ', retry #%d' % i
1513             webpage = self._download_webpage(url, title, msg)
1514             ids = orderedSet(re.findall(video_re, webpage))
1515             if ids:
1516                 break
1517         url_results = self._ids_to_results(ids)
1518         return self.playlist_result(url_results, playlist_title=title)
1519
1520
1521 class YoutubeChannelIE(InfoExtractor):
1522     IE_DESC = u'YouTube.com channels'
1523     _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
1524     _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
1525     _MORE_PAGES_URL = 'https://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
1526     IE_NAME = u'youtube:channel'
1527
1528     def extract_videos_from_page(self, page):
1529         ids_in_page = []
1530         for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
1531             if mobj.group(1) not in ids_in_page:
1532                 ids_in_page.append(mobj.group(1))
1533         return ids_in_page
1534
1535     def _real_extract(self, url):
1536         # Extract channel id
1537         mobj = re.match(self._VALID_URL, url)
1538         if mobj is None:
1539             raise ExtractorError(u'Invalid URL: %s' % url)
1540
1541         # Download channel page
1542         channel_id = mobj.group(1)
1543         video_ids = []
1544         url = 'https://www.youtube.com/channel/%s/videos' % channel_id
1545         channel_page = self._download_webpage(url, channel_id)
1546         autogenerated = re.search(r'''(?x)
1547                 class="[^"]*?(?:
1548                     channel-header-autogenerated-label|
1549                     yt-channel-title-autogenerated
1550                 )[^"]*"''', channel_page) is not None
1551
1552         if autogenerated:
1553             # The videos are contained in a single page
1554             # the ajax pages can't be used, they are empty
1555             video_ids = self.extract_videos_from_page(channel_page)
1556         else:
1557             # Download all channel pages using the json-based channel_ajax query
1558             for pagenum in itertools.count(1):
1559                 url = self._MORE_PAGES_URL % (pagenum, channel_id)
1560                 page = self._download_json(
1561                     url, channel_id, note=u'Downloading page #%s' % pagenum,
1562                     transform_source=uppercase_escape)
1563
1564                 ids_in_page = self.extract_videos_from_page(page['content_html'])
1565                 video_ids.extend(ids_in_page)
1566
1567                 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
1568                     break
1569
1570         self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
1571
1572         url_entries = [self.url_result(video_id, 'Youtube', video_id=video_id)
1573                        for video_id in video_ids]
1574         return self.playlist_result(url_entries, channel_id)
1575
1576
1577 class YoutubeUserIE(InfoExtractor):
1578     IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
1579     _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
1580     _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/users/%s'
1581     _GDATA_PAGE_SIZE = 50
1582     _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
1583     IE_NAME = u'youtube:user'
1584
1585     @classmethod
1586     def suitable(cls, url):
1587         # Don't return True if the url can be extracted with other youtube
1588         # extractor, the regex would is too permissive and it would match.
1589         other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
1590         if any(ie.suitable(url) for ie in other_ies): return False
1591         else: return super(YoutubeUserIE, cls).suitable(url)
1592
1593     def _real_extract(self, url):
1594         # Extract username
1595         mobj = re.match(self._VALID_URL, url)
1596         if mobj is None:
1597             raise ExtractorError(u'Invalid URL: %s' % url)
1598
1599         username = mobj.group(1)
1600
1601         # Download video ids using YouTube Data API. Result size per
1602         # query is limited (currently to 50 videos) so we need to query
1603         # page by page until there are no video ids - it means we got
1604         # all of them.
1605
1606         def download_page(pagenum):
1607             start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1608
1609             gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
1610             page = self._download_webpage(
1611                 gdata_url, username,
1612                 u'Downloading video ids from %d to %d' % (
1613                     start_index, start_index + self._GDATA_PAGE_SIZE))
1614
1615             try:
1616                 response = json.loads(page)
1617             except ValueError as err:
1618                 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
1619             if 'entry' not in response['feed']:
1620                 return
1621
1622             # Extract video identifiers
1623             entries = response['feed']['entry']
1624             for entry in entries:
1625                 title = entry['title']['$t']
1626                 video_id = entry['id']['$t'].split('/')[-1]
1627                 yield {
1628                     '_type': 'url',
1629                     'url': video_id,
1630                     'ie_key': 'Youtube',
1631                     'id': video_id,
1632                     'title': title,
1633                 }
1634         url_results = PagedList(download_page, self._GDATA_PAGE_SIZE)
1635
1636         return self.playlist_result(url_results, playlist_title=username)
1637
1638
1639 class YoutubeSearchIE(SearchInfoExtractor):
1640     IE_DESC = u'YouTube.com searches'
1641     _API_URL = u'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1642     _MAX_RESULTS = 1000
1643     IE_NAME = u'youtube:search'
1644     _SEARCH_KEY = 'ytsearch'
1645
1646     def _get_n_results(self, query, n):
1647         """Get a specified number of results for a query"""
1648
1649         video_ids = []
1650         pagenum = 0
1651         limit = n
1652         PAGE_SIZE = 50
1653
1654         while (PAGE_SIZE * pagenum) < limit:
1655             result_url = self._API_URL % (
1656                 compat_urllib_parse.quote_plus(query.encode('utf-8')),
1657                 (PAGE_SIZE * pagenum) + 1)
1658             data_json = self._download_webpage(
1659                 result_url, video_id=u'query "%s"' % query,
1660                 note=u'Downloading page %s' % (pagenum + 1),
1661                 errnote=u'Unable to download API page')
1662             data = json.loads(data_json)
1663             api_response = data['data']
1664
1665             if 'items' not in api_response:
1666                 raise ExtractorError(
1667                     u'[youtube] No video results', expected=True)
1668
1669             new_ids = list(video['id'] for video in api_response['items'])
1670             video_ids += new_ids
1671
1672             limit = min(n, api_response['totalItems'])
1673             pagenum += 1
1674
1675         if len(video_ids) > n:
1676             video_ids = video_ids[:n]
1677         videos = [self.url_result(video_id, 'Youtube', video_id=video_id)
1678                   for video_id in video_ids]
1679         return self.playlist_result(videos, query)
1680
1681
1682 class YoutubeSearchDateIE(YoutubeSearchIE):
1683     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
1684     _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
1685     _SEARCH_KEY = 'ytsearchdate'
1686     IE_DESC = u'YouTube.com searches, newest videos first'
1687
1688
1689 class YoutubeSearchURLIE(InfoExtractor):
1690     IE_DESC = u'YouTube.com search URLs'
1691     IE_NAME = u'youtube:search_url'
1692     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?search_query=(?P<query>[^&]+)(?:[&]|$)'
1693
1694     def _real_extract(self, url):
1695         mobj = re.match(self._VALID_URL, url)
1696         query = compat_urllib_parse.unquote_plus(mobj.group('query'))
1697
1698         webpage = self._download_webpage(url, query)
1699         result_code = self._search_regex(
1700             r'(?s)<ol id="search-results"(.*?)</ol>', webpage, u'result HTML')
1701
1702         part_codes = re.findall(
1703             r'(?s)<h3 class="yt-lockup-title">(.*?)</h3>', result_code)
1704         entries = []
1705         for part_code in part_codes:
1706             part_title = self._html_search_regex(
1707                 r'(?s)title="([^"]+)"', part_code, 'item title', fatal=False)
1708             part_url_snippet = self._html_search_regex(
1709                 r'(?s)href="([^"]+)"', part_code, 'item URL')
1710             part_url = compat_urlparse.urljoin(
1711                 'https://www.youtube.com/', part_url_snippet)
1712             entries.append({
1713                 '_type': 'url',
1714                 'url': part_url,
1715                 'title': part_title,
1716             })
1717
1718         return {
1719             '_type': 'playlist',
1720             'entries': entries,
1721             'title': query,
1722         }
1723
1724
1725 class YoutubeShowIE(InfoExtractor):
1726     IE_DESC = u'YouTube.com (multi-season) shows'
1727     _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
1728     IE_NAME = u'youtube:show'
1729
1730     def _real_extract(self, url):
1731         mobj = re.match(self._VALID_URL, url)
1732         show_name = mobj.group(1)
1733         webpage = self._download_webpage(url, show_name, u'Downloading show webpage')
1734         # There's one playlist for each season of the show
1735         m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
1736         self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
1737         return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
1738
1739
1740 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
1741     """
1742     Base class for extractors that fetch info from
1743     http://www.youtube.com/feed_ajax
1744     Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
1745     """
1746     _LOGIN_REQUIRED = True
1747     # use action_load_personal_feed instead of action_load_system_feed
1748     _PERSONAL_FEED = False
1749
1750     @property
1751     def _FEED_TEMPLATE(self):
1752         action = 'action_load_system_feed'
1753         if self._PERSONAL_FEED:
1754             action = 'action_load_personal_feed'
1755         return 'https://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
1756
1757     @property
1758     def IE_NAME(self):
1759         return u'youtube:%s' % self._FEED_NAME
1760
1761     def _real_initialize(self):
1762         self._login()
1763
1764     def _real_extract(self, url):
1765         feed_entries = []
1766         paging = 0
1767         for i in itertools.count(1):
1768             info = self._download_json(self._FEED_TEMPLATE % paging,
1769                                           u'%s feed' % self._FEED_NAME,
1770                                           u'Downloading page %s' % i)
1771             feed_html = info.get('feed_html') or info.get('content_html')
1772             m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
1773             ids = orderedSet(m.group(1) for m in m_ids)
1774             feed_entries.extend(
1775                 self.url_result(video_id, 'Youtube', video_id=video_id)
1776                 for video_id in ids)
1777             mobj = re.search(
1778                 r'data-uix-load-more-href="/?[^"]+paging=(?P<paging>\d+)',
1779                 feed_html)
1780             if mobj is None:
1781                 break
1782             paging = mobj.group('paging')
1783         return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
1784
1785 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
1786     IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
1787     _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
1788     _FEED_NAME = 'subscriptions'
1789     _PLAYLIST_TITLE = u'Youtube Subscriptions'
1790
1791 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
1792     IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
1793     _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
1794     _FEED_NAME = 'recommended'
1795     _PLAYLIST_TITLE = u'Youtube Recommended videos'
1796
1797 class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
1798     IE_DESC = u'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
1799     _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
1800     _FEED_NAME = 'watch_later'
1801     _PLAYLIST_TITLE = u'Youtube Watch Later'
1802     _PERSONAL_FEED = True
1803
1804 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
1805     IE_DESC = u'Youtube watch history, "ythistory" keyword (requires authentication)'
1806     _VALID_URL = u'https?://www\.youtube\.com/feed/history|:ythistory'
1807     _FEED_NAME = 'history'
1808     _PERSONAL_FEED = True
1809     _PLAYLIST_TITLE = u'Youtube Watch History'
1810
1811 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
1812     IE_NAME = u'youtube:favorites'
1813     IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'
1814     _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
1815     _LOGIN_REQUIRED = True
1816
1817     def _real_extract(self, url):
1818         webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
1819         playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, u'favourites playlist id')
1820         return self.url_result(playlist_id, 'YoutubePlaylist')
1821
1822
1823 class YoutubeTruncatedURLIE(InfoExtractor):
1824     IE_NAME = 'youtube:truncated_url'
1825     IE_DESC = False  # Do not list
1826     _VALID_URL = r'''(?x)
1827         (?:https?://)?[^/]+/watch\?(?:feature=[a-z_]+)?$|
1828         (?:https?://)?(?:www\.)?youtube\.com/attribution_link\?a=[^&]+$
1829     '''
1830
1831     def _real_extract(self, url):
1832         raise ExtractorError(
1833             u'Did you forget to quote the URL? Remember that & is a meta '
1834             u'character in most shells, so you want to put the URL in quotes, '
1835             u'like  youtube-dl '
1836             u'"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
1837             u' or simply  youtube-dl BaW_jenozKc  .',
1838             expected=True)