Raphaël G. Git Repositories - youtubedl/blob - youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # License: Public domain code
   6 import htmlentitydefs
   7 import httplib
   8 import locale
   9 import math
  10 import netrc
  11 import os
  12 import os.path
  13 import re
  14 import socket
  15 import string
  16 import sys
  17 import time
  18 import urllib
  19 import urllib2
  20
  21 std_headers = {
  22         'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2',
  23         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  24         'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
  25         'Accept-Language': 'en-us,en;q=0.5',
  26 }
  27
  28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  29
  30 def preferredencoding():
  31         """Get preferred encoding.
  32
  33         Returns the best encoding scheme for the system, based on
  34         locale.getpreferredencoding() and some further tweaks.
  35         """
  36         try:
  37                 pref = locale.getpreferredencoding()
  38                 # Mac OSX systems have this problem sometimes
  39                 if pref == '':
  40                         return 'UTF-8'
  41                 return pref
  42         except:
  43                 sys.stderr.write('WARNING: problem obtaining preferred encoding. Falling back to UTF-8.\n')
  44                 return 'UTF-8'
  45
  46 class DownloadError(Exception):
  47         """Download Error exception.
  48
  49         This exception may be thrown by FileDownloader objects if they are not
  50         configured to continue on errors. They will contain the appropriate
  51         error message.
  52         """
  53         pass
  54
  55 class SameFileError(Exception):
  56         """Same File exception.
  57
  58         This exception will be thrown by FileDownloader objects if they detect
  59         multiple files would have to be downloaded to the same file on disk.
  60         """
  61         pass
  62
  63 class PostProcessingError(Exception):
  64         """Post Processing exception.
  65
  66         This exception may be raised by PostProcessor's .run() method to
  67         indicate an error in the postprocessing task.
  68         """
  69         pass
  70
  71 class UnavailableFormatError(Exception):
  72         """Unavailable Format exception.
  73
  74         This exception will be thrown when a video is requested
  75         in a format that is not available for that video.
  76         """
  77         pass
  78
  79 class ContentTooShortError(Exception):
  80         """Content Too Short exception.
  81
  82         This exception may be raised by FileDownloader objects when a file they
  83         download is too small for what the server announced first, indicating
  84         the connection was probably interrupted.
  85         """
  86         # Both in bytes
  87         downloaded = None
  88         expected = None
  89
  90         def __init__(self, downloaded, expected):
  91                 self.downloaded = downloaded
  92                 self.expected = expected
  93
  94 class FileDownloader(object):
  95         """File Downloader class.
  96
  97         File downloader objects are the ones responsible of downloading the
  98         actual video file and writing it to disk if the user has requested
  99         it, among some other tasks. In most cases there should be one per
 100         program. As, given a video URL, the downloader doesn't know how to
 101         extract all the needed information, task that InfoExtractors do, it
 102         has to pass the URL to one of them.
 103
 104         For this, file downloader objects have a method that allows
 105         InfoExtractors to be registered in a given order. When it is passed
 106         a URL, the file downloader handles it to the first InfoExtractor it
 107         finds that reports being able to handle it. The InfoExtractor extracts
 108         all the information about the video or videos the URL refers to, and
 109         asks the FileDownloader to process the video information, possibly
 110         downloading the video.
 111
 112         File downloaders accept a lot of parameters. In order not to saturate
 113         the object constructor with arguments, it receives a dictionary of
 114         options instead. These options are available through the params
 115         attribute for the InfoExtractors to use. The FileDownloader also
 116         registers itself as the downloader in charge for the InfoExtractors
 117         that are added to it, so this is a "mutual registration".
 118
 119         Available options:
 120
 121         username:       Username for authentication purposes.
 122         password:       Password for authentication purposes.
 123         usenetrc:       Use netrc for authentication instead.
 124         quiet:          Do not print messages to stdout.
 125         forceurl:       Force printing final URL.
 126         forcetitle:     Force printing title.
 127         simulate:       Do not download the video files.
 128         format:         Video format code.
 129         outtmpl:        Template for output names.
 130         ignoreerrors:   Do not stop on download errors.
 131         ratelimit:      Download speed limit, in bytes/sec.
 132         nooverwrites:   Prevent overwriting files.
 133         continuedl:     Try to continue downloads if possible.
 134         """
 135
 136         params = None
 137         _ies = []
 138         _pps = []
 139         _download_retcode = None
 140
 141         def __init__(self, params):
 142                 """Create a FileDownloader object with the given options."""
 143                 self._ies = []
 144                 self._pps = []
 145                 self._download_retcode = 0
 146                 self.params = params
 147
 148         @staticmethod
 149         def pmkdir(filename):
 150                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 151                 components = filename.split(os.sep)
 152                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 153                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 154                 for dir in aggregate:
 155                         if not os.path.exists(dir):
 156                                 os.mkdir(dir)
 157
 158         @staticmethod
 159         def format_bytes(bytes):
 160                 if bytes is None:
 161                         return 'N/A'
 162                 if type(bytes) is str:
 163                         bytes = float(bytes)
 164                 if bytes == 0.0:
 165                         exponent = 0
 166                 else:
 167                         exponent = long(math.log(bytes, 1024.0))
 168                 suffix = 'bkMGTPEZY'[exponent]
 169                 converted = float(bytes) / float(1024**exponent)
 170                 return '%.2f%s' % (converted, suffix)
 171
 172         @staticmethod
 173         def calc_percent(byte_counter, data_len):
 174                 if data_len is None:
 175                         return '---.-%'
 176                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 177
 178         @staticmethod
 179         def calc_eta(start, now, total, current):
 180                 if total is None:
 181                         return '--:--'
 182                 dif = now - start
 183                 if current == 0 or dif < 0.001: # One millisecond
 184                         return '--:--'
 185                 rate = float(current) / dif
 186                 eta = long((float(total) - float(current)) / rate)
 187                 (eta_mins, eta_secs) = divmod(eta, 60)
 188                 if eta_mins > 99:
 189                         return '--:--'
 190                 return '%02d:%02d' % (eta_mins, eta_secs)
 191
 192         @staticmethod
 193         def calc_speed(start, now, bytes):
 194                 dif = now - start
 195                 if bytes == 0 or dif < 0.001: # One millisecond
 196                         return '%10s' % '---b/s'
 197                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 198
 199         @staticmethod
 200         def best_block_size(elapsed_time, bytes):
 201                 new_min = max(bytes / 2.0, 1.0)
 202                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 203                 if elapsed_time < 0.001:
 204                         return long(new_max)
 205                 rate = bytes / elapsed_time
 206                 if rate > new_max:
 207                         return long(new_max)
 208                 if rate < new_min:
 209                         return long(new_min)
 210                 return long(rate)
 211
 212         @staticmethod
 213         def parse_bytes(bytestr):
 214                 """Parse a string indicating a byte quantity into a long integer."""
 215                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 216                 if matchobj is None:
 217                         return None
 218                 number = float(matchobj.group(1))
 219                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 220                 return long(round(number * multiplier))
 221
 222         @staticmethod
 223         def verify_url(url):
 224                 """Verify a URL is valid and data could be downloaded. Return real data URL."""
 225                 request = urllib2.Request(url, None, std_headers)
 226                 data = urllib2.urlopen(request)
 227                 data.read(1)
 228                 url = data.geturl()
 229                 data.close()
 230                 return url
 231
 232         def add_info_extractor(self, ie):
 233                 """Add an InfoExtractor object to the end of the list."""
 234                 self._ies.append(ie)
 235                 ie.set_downloader(self)
 236
 237         def add_post_processor(self, pp):
 238                 """Add a PostProcessor object to the end of the chain."""
 239                 self._pps.append(pp)
 240                 pp.set_downloader(self)
 241
 242         def to_stdout(self, message, skip_eol=False):
 243                 """Print message to stdout if not in quiet mode."""
 244                 if not self.params.get('quiet', False):
 245                         print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
 246                         sys.stdout.flush()
 247
 248         def to_stderr(self, message):
 249                 """Print message to stderr."""
 250                 print >>sys.stderr, message.encode(preferredencoding())
 251
 252         def fixed_template(self):
 253                 """Checks if the output template is fixed."""
 254                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 255
 256         def trouble(self, message=None):
 257                 """Determine action to take when a download problem appears.
 258
 259                 Depending on if the downloader has been configured to ignore
 260                 download errors or not, this method may throw an exception or
 261                 not when errors are found, after printing the message.
 262                 """
 263                 if message is not None:
 264                         self.to_stderr(message)
 265                 if not self.params.get('ignoreerrors', False):
 266                         raise DownloadError(message)
 267                 self._download_retcode = 1
 268
 269         def slow_down(self, start_time, byte_counter):
 270                 """Sleep if the download speed is over the rate limit."""
 271                 rate_limit = self.params.get('ratelimit', None)
 272                 if rate_limit is None or byte_counter == 0:
 273                         return
 274                 now = time.time()
 275                 elapsed = now - start_time
 276                 if elapsed <= 0.0:
 277                         return
 278                 speed = float(byte_counter) / elapsed
 279                 if speed > rate_limit:
 280                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 281
 282         def report_destination(self, filename):
 283                 """Report destination filename."""
 284                 self.to_stdout(u'[download] Destination: %s' % filename)
 285
 286         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 287                 """Report download progress."""
 288                 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
 289                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 290
 291         def report_resuming_byte(self, resume_len):
 292                 """Report attemtp to resume at given byte."""
 293                 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
 294
 295         def report_file_already_downloaded(self, file_name):
 296                 """Report file has already been fully downloaded."""
 297                 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
 298
 299         def report_unable_to_resume(self):
 300                 """Report it was impossible to resume download."""
 301                 self.to_stdout(u'[download] Unable to resume')
 302
 303         def report_finish(self):
 304                 """Report download finished."""
 305                 self.to_stdout(u'')
 306
 307         def process_info(self, info_dict):
 308                 """Process a single dictionary returned by an InfoExtractor."""
 309                 # Do nothing else if in simulate mode
 310                 if self.params.get('simulate', False):
 311                         try:
 312                                 info_dict['url'] = self.verify_url(info_dict['url'])
 313                         except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
 314                                 raise UnavailableFormatError
 315
 316                         # Forced printings
 317                         if self.params.get('forcetitle', False):
 318                                 print info_dict['title'].encode(preferredencoding())
 319                         if self.params.get('forceurl', False):
 320                                 print info_dict['url'].encode(preferredencoding())
 321
 322                         return
 323
 324                 try:
 325                         template_dict = dict(info_dict)
 326                         template_dict['epoch'] = unicode(long(time.time()))
 327                         filename = self.params['outtmpl'] % template_dict
 328                 except (ValueError, KeyError), err:
 329                         self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
 330                 if self.params['nooverwrites'] and os.path.exists(filename):
 331                         self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
 332                         return
 333
 334                 try:
 335                         self.pmkdir(filename)
 336                 except (OSError, IOError), err:
 337                         self.trouble('ERROR: unable to create directories: %s' % str(err))
 338                         return
 339
 340                 try:
 341                         success = self._do_download(filename, info_dict['url'])
 342                 except (OSError, IOError), err:
 343                         raise UnavailableFormatError
 344                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 345                         self.trouble('ERROR: unable to download video data: %s' % str(err))
 346                         return
 347                 except (ContentTooShortError, ), err:
 348                         self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 349                         return
 350
 351                 if success:
 352                         try:
 353                                 self.post_process(filename, info_dict)
 354                         except (PostProcessingError), err:
 355                                 self.trouble('ERROR: postprocessing: %s' % str(err))
 356                                 return
 357
 358         def download(self, url_list):
 359                 """Download a given list of URLs."""
 360                 if len(url_list) > 1 and self.fixed_template():
 361                         raise SameFileError(self.params['outtmpl'])
 362
 363                 for url in url_list:
 364                         suitable_found = False
 365                         for ie in self._ies:
 366                                 # Go to next InfoExtractor if not suitable
 367                                 if not ie.suitable(url):
 368                                         continue
 369
 370                                 # Suitable InfoExtractor found
 371                                 suitable_found = True
 372
 373                                 # Extract information from URL and process it
 374                                 ie.extract(url)
 375
 376                                 # Suitable InfoExtractor had been found; go to next URL
 377                                 break
 378
 379                         if not suitable_found:
 380                                 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
 381
 382                 return self._download_retcode
 383
 384         def post_process(self, filename, ie_info):
 385                 """Run the postprocessing chain on the given file."""
 386                 info = dict(ie_info)
 387                 info['filepath'] = filename
 388                 for pp in self._pps:
 389                         info = pp.run(info)
 390                         if info is None:
 391                                 break
 392
 393         def _do_download(self, filename, url):
 394                 stream = None
 395                 open_mode = 'ab'
 396
 397                 basic_request = urllib2.Request(url, None, std_headers)
 398                 request = urllib2.Request(url, None, std_headers)
 399
 400                 # Attempt to resume download with "continuedl" option
 401                 if os.path.isfile(filename):
 402                         resume_len = os.path.getsize(filename)
 403                 else:
 404                         resume_len = 0
 405                 if self.params['continuedl'] and resume_len != 0:
 406                         self.report_resuming_byte(resume_len)
 407                         request.add_header('Range','bytes=%d-' % resume_len)
 408
 409                 # Establish connection
 410                 try:
 411                         data = urllib2.urlopen(request)
 412                 except (urllib2.HTTPError, ), err:
 413                         if err.code != 416: #  416 is 'Requested range not satisfiable'
 414                                 raise
 415                         data = urllib2.urlopen(basic_request)
 416                         content_length = data.info()['Content-Length']
 417                         if content_length is not None and long(content_length) == resume_len:
 418                                 self.report_file_already_downloaded(filename)
 419                                 return True
 420                         else:
 421                                 self.report_unable_to_resume()
 422                                 open_mode = 'wb'
 423
 424                 data_len = data.info().get('Content-length', None)
 425                 data_len_str = self.format_bytes(data_len)
 426                 byte_counter = 0
 427                 block_size = 1024
 428                 start = time.time()
 429                 while True:
 430                         # Download and write
 431                         before = time.time()
 432                         data_block = data.read(block_size)
 433                         after = time.time()
 434                         data_block_len = len(data_block)
 435                         if data_block_len == 0:
 436                                 break
 437                         byte_counter += data_block_len
 438
 439                         # Open file just in time
 440                         if stream is None:
 441                                 try:
 442                                         stream = open(filename, open_mode)
 443                                         self.report_destination(filename)
 444                                 except (OSError, IOError), err:
 445                                         self.trouble('ERROR: unable to open for writing: %s' % str(err))
 446                                         return False
 447                         stream.write(data_block)
 448                         block_size = self.best_block_size(after - before, data_block_len)
 449
 450                         # Progress message
 451                         percent_str = self.calc_percent(byte_counter, data_len)
 452                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 453                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 454                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 455
 456                         # Apply rate limit
 457                         self.slow_down(start, byte_counter)
 458
 459                 self.report_finish()
 460                 if data_len is not None and str(byte_counter) != data_len:
 461                         raise ContentTooShortError(byte_counter, long(data_len))
 462                 return True
 463
 464 class InfoExtractor(object):
 465         """Information Extractor class.
 466
 467         Information extractors are the classes that, given a URL, extract
 468         information from the video (or videos) the URL refers to. This
 469         information includes the real video URL, the video title and simplified
 470         title, author and others. The information is stored in a dictionary
 471         which is then passed to the FileDownloader. The FileDownloader
 472         processes this information possibly downloading the video to the file
 473         system, among other possible outcomes. The dictionaries must include
 474         the following fields:
 475
 476         id:             Video identifier.
 477         url:            Final video URL.
 478         uploader:       Nickname of the video uploader.
 479         title:          Literal title.
 480         stitle:         Simplified title.
 481         ext:            Video filename extension.
 482
 483         Subclasses of this one should re-define the _real_initialize() and
 484         _real_extract() methods, as well as the suitable() static method.
 485         Probably, they should also be instantiated and added to the main
 486         downloader.
 487         """
 488
 489         _ready = False
 490         _downloader = None
 491
 492         def __init__(self, downloader=None):
 493                 """Constructor. Receives an optional downloader."""
 494                 self._ready = False
 495                 self.set_downloader(downloader)
 496
 497         @staticmethod
 498         def suitable(url):
 499                 """Receives a URL and returns True if suitable for this IE."""
 500                 return False
 501
 502         def initialize(self):
 503                 """Initializes an instance (authentication, etc)."""
 504                 if not self._ready:
 505                         self._real_initialize()
 506                         self._ready = True
 507
 508         def extract(self, url):
 509                 """Extracts URL information and returns it in list of dicts."""
 510                 self.initialize()
 511                 return self._real_extract(url)
 512
 513         def set_downloader(self, downloader):
 514                 """Sets the downloader for this IE."""
 515                 self._downloader = downloader
 516
 517         def _real_initialize(self):
 518                 """Real initialization process. Redefine in subclasses."""
 519                 pass
 520
 521         def _real_extract(self, url):
 522                 """Real extraction process. Redefine in subclasses."""
 523                 pass
 524
 525 class YoutubeIE(InfoExtractor):
 526         """Information extractor for youtube.com."""
 527
 528         _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
 529         _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 530         _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
 531         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 532         _NETRC_MACHINE = 'youtube'
 533         _available_formats = ['22', '35', '18', '5', '17', '13', None] # listed in order of priority for -b flag
 534         _video_extensions = {
 535                 '13': '3gp',
 536                 '17': 'mp4',
 537                 '18': 'mp4',
 538                 '22': 'mp4',
 539         }
 540
 541         @staticmethod
 542         def suitable(url):
 543                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 544
 545         @staticmethod
 546         def htmlentity_transform(matchobj):
 547                 """Transforms an HTML entity to a Unicode character."""
 548                 entity = matchobj.group(1)
 549
 550                 # Known non-numeric HTML entity
 551                 if entity in htmlentitydefs.name2codepoint:
 552                         return unichr(htmlentitydefs.name2codepoint[entity])
 553
 554                 # Unicode character
 555                 mobj = re.match(ur'(?u)#(x?\d+)', entity)
 556                 if mobj is not None:
 557                         numstr = mobj.group(1)
 558                         if numstr.startswith(u'x'):
 559                                 base = 16
 560                                 numstr = u'0%s' % numstr
 561                         else:
 562                                 base = 10
 563                         return unichr(long(numstr, base))
 564
 565                 # Unknown entity in name, return its literal representation
 566                 return (u'&%s;' % entity)
 567
 568         def report_lang(self):
 569                 """Report attempt to set language."""
 570                 self._downloader.to_stdout(u'[youtube] Setting language')
 571
 572         def report_login(self):
 573                 """Report attempt to log in."""
 574                 self._downloader.to_stdout(u'[youtube] Logging in')
 575
 576         def report_age_confirmation(self):
 577                 """Report attempt to confirm age."""
 578                 self._downloader.to_stdout(u'[youtube] Confirming age')
 579
 580         def report_video_info_webpage_download(self, video_id):
 581                 """Report attempt to download video info webpage."""
 582                 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
 583
 584         def report_information_extraction(self, video_id):
 585                 """Report attempt to extract video information."""
 586                 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
 587
 588         def report_unavailable_format(self, video_id, format):
 589                 """Report extracted video URL."""
 590                 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
 591
 592         def _real_initialize(self):
 593                 if self._downloader is None:
 594                         return
 595
 596                 username = None
 597                 password = None
 598                 downloader_params = self._downloader.params
 599
 600                 # Attempt to use provided username and password or .netrc data
 601                 if downloader_params.get('username', None) is not None:
 602                         username = downloader_params['username']
 603                         password = downloader_params['password']
 604                 elif downloader_params.get('usenetrc', False):
 605                         try:
 606                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 607                                 if info is not None:
 608                                         username = info[0]
 609                                         password = info[2]
 610                                 else:
 611                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 612                         except (IOError, netrc.NetrcParseError), err:
 613                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 614                                 return
 615
 616                 # Set language
 617                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 618                 try:
 619                         self.report_lang()
 620                         urllib2.urlopen(request).read()
 621                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 622                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 623                         return
 624
 625                 # No authentication to be performed
 626                 if username is None:
 627                         return
 628
 629                 # Log in
 630                 login_form = {
 631                                 'current_form': 'loginForm',
 632                                 'next':         '/',
 633                                 'action_login': 'Log In',
 634                                 'username':     username,
 635                                 'password':     password,
 636                                 }
 637                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 638                 try:
 639                         self.report_login()
 640                         login_results = urllib2.urlopen(request).read()
 641                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 642                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 643                                 return
 644                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 645                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 646                         return
 647
 648                 # Confirm age
 649                 age_form = {
 650                                 'next_url':             '/',
 651                                 'action_confirm':       'Confirm',
 652                                 }
 653                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 654                 try:
 655                         self.report_age_confirmation()
 656                         age_results = urllib2.urlopen(request).read()
 657                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 658                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 659                         return
 660
 661         def _real_extract(self, url):
 662                 # Extract video id from URL
 663                 mobj = re.match(self._VALID_URL, url)
 664                 if mobj is None:
 665                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 666                         return
 667                 video_id = mobj.group(2)
 668
 669                 # Downloader parameters
 670                 best_quality = False
 671                 format_param = None
 672                 quality_index = 0
 673                 if self._downloader is not None:
 674                         params = self._downloader.params
 675                         format_param = params.get('format', None)
 676                         if format_param == '0':
 677                                 format_param = self._available_formats[quality_index]
 678                                 best_quality = True
 679
 680                 while True:
 681                         # Extension
 682                         video_extension = self._video_extensions.get(format_param, 'flv')
 683
 684                         # Get video info
 685                         video_info_url = 'http://www.youtube.com/get_video_info?&video_id=%s&el=detailpage&ps=default&eurl=&gl=US&hl=en' % video_id
 686                         request = urllib2.Request(video_info_url, None, std_headers)
 687                         try:
 688                                 self.report_video_info_webpage_download(video_id)
 689                                 video_info_webpage = urllib2.urlopen(request).read()
 690                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 691                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 692                                 return
 693                         self.report_information_extraction(video_id)
 694
 695                         # "t" param
 696                         mobj = re.search(r'(?m)&token=([^&]+)(?:&|$)', video_info_webpage)
 697                         if mobj is None:
 698                                 # Attempt to see if YouTube has issued an error message
 699                                 mobj = re.search(r'(?m)&reason=([^&]+)(?:&|$)', video_info_webpage)
 700                                 if mobj is None:
 701                                         self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
 702                                         stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
 703                                         stream.write(video_info_webpage)
 704                                         stream.close()
 705                                 else:
 706                                         reason = urllib.unquote_plus(mobj.group(1))
 707                                         self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
 708                                 return
 709                         token = urllib.unquote(mobj.group(1))
 710                         video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
 711                         if format_param is not None:
 712                                 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
 713
 714                         # uploader
 715                         mobj = re.search(r'(?m)&author=([^&]+)(?:&|$)', video_info_webpage)
 716                         if mobj is None:
 717                                 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 718                                 return
 719                         video_uploader = urllib.unquote(mobj.group(1))
 720
 721                         # title
 722                         mobj = re.search(r'(?m)&title=([^&]+)(?:&|$)', video_info_webpage)
 723                         if mobj is None:
 724                                 self._downloader.trouble(u'ERROR: unable to extract video title')
 725                                 return
 726                         video_title = urllib.unquote(mobj.group(1))
 727                         video_title = video_title.decode('utf-8')
 728                         video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
 729                         video_title = video_title.replace(os.sep, u'%')
 730
 731                         # simplified title
 732                         simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 733                         simple_title = simple_title.strip(ur'_')
 734
 735                         try:
 736                                 # Process video information
 737                                 self._downloader.process_info({
 738                                         'id':           video_id.decode('utf-8'),
 739                                         'url':          video_real_url.decode('utf-8'),
 740                                         'uploader':     video_uploader.decode('utf-8'),
 741                                         'title':        video_title,
 742                                         'stitle':       simple_title,
 743                                         'ext':          video_extension.decode('utf-8'),
 744                                 })
 745
 746                                 return
 747
 748                         except UnavailableFormatError, err:
 749                                 if best_quality:
 750                                         if quality_index == len(self._available_formats) - 1:
 751                                                 # I don't ever expect this to happen
 752                                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 753                                                 return
 754                                         else:
 755                                                 self.report_unavailable_format(video_id, format_param)
 756                                                 quality_index += 1
 757                                                 format_param = self._available_formats[quality_index]
 758                                                 continue
 759                                 else:
 760                                         self._downloader.trouble('ERROR: format not available for video')
 761                                         return
 762
 763
 764 class MetacafeIE(InfoExtractor):
 765         """Information Extractor for metacafe.com."""
 766
 767         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 768         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 769         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 770         _youtube_ie = None
 771
 772         def __init__(self, youtube_ie, downloader=None):
 773                 InfoExtractor.__init__(self, downloader)
 774                 self._youtube_ie = youtube_ie
 775
 776         @staticmethod
 777         def suitable(url):
 778                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
 779
 780         def report_disclaimer(self):
 781                 """Report disclaimer retrieval."""
 782                 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
 783
 784         def report_age_confirmation(self):
 785                 """Report attempt to confirm age."""
 786                 self._downloader.to_stdout(u'[metacafe] Confirming age')
 787
 788         def report_download_webpage(self, video_id):
 789                 """Report webpage download."""
 790                 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
 791
 792         def report_extraction(self, video_id):
 793                 """Report information extraction."""
 794                 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
 795
 796         def _real_initialize(self):
 797                 # Retrieve disclaimer
 798                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
 799                 try:
 800                         self.report_disclaimer()
 801                         disclaimer = urllib2.urlopen(request).read()
 802                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 803                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
 804                         return
 805
 806                 # Confirm age
 807                 disclaimer_form = {
 808                         'filters': '0',
 809                         'submit': "Continue - I'm over 18",
 810                         }
 811                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
 812                 try:
 813                         self.report_age_confirmation()
 814                         disclaimer = urllib2.urlopen(request).read()
 815                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 816                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 817                         return
 818
 819         def _real_extract(self, url):
 820                 # Extract id and simplified title from URL
 821                 mobj = re.match(self._VALID_URL, url)
 822                 if mobj is None:
 823                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 824                         return
 825
 826                 video_id = mobj.group(1)
 827
 828                 # Check if video comes from YouTube
 829                 mobj2 = re.match(r'^yt-(.*)$', video_id)
 830                 if mobj2 is not None:
 831                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
 832                         return
 833
 834                 simple_title = mobj.group(2).decode('utf-8')
 835                 video_extension = 'flv'
 836
 837                 # Retrieve video webpage to extract further information
 838                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 839                 try:
 840                         self.report_download_webpage(video_id)
 841                         webpage = urllib2.urlopen(request).read()
 842                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 843                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 844                         return
 845
 846                 # Extract URL, uploader and title from webpage
 847                 self.report_extraction(video_id)
 848                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
 849                 if mobj is None:
 850                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 851                         return
 852                 mediaURL = urllib.unquote(mobj.group(1))
 853
 854                 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 855                 #if mobj is None:
 856                 #       self._downloader.trouble(u'ERROR: unable to extract gdaKey')
 857                 #       return
 858                 #gdaKey = mobj.group(1)
 859                 #
 860                 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 861
 862                 video_url = mediaURL
 863
 864                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 865                 if mobj is None:
 866                         self._downloader.trouble(u'ERROR: unable to extract title')
 867                         return
 868                 video_title = mobj.group(1).decode('utf-8')
 869
 870                 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
 871                 if mobj is None:
 872                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 873                         return
 874                 video_uploader = mobj.group(1)
 875
 876                 try:
 877                         # Process video information
 878                         self._downloader.process_info({
 879                                 'id':           video_id.decode('utf-8'),
 880                                 'url':          video_url.decode('utf-8'),
 881                                 'uploader':     video_uploader.decode('utf-8'),
 882                                 'title':        video_title,
 883                                 'stitle':       simple_title,
 884                                 'ext':          video_extension.decode('utf-8'),
 885                         })
 886                 except UnavailableFormatError:
 887                         self._downloader.trouble(u'ERROR: format not available for video')
 888
 889
 890 class YoutubeSearchIE(InfoExtractor):
 891         """Information Extractor for YouTube search queries."""
 892         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
 893         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
 894         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
 895         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
 896         _youtube_ie = None
 897         _max_youtube_results = 1000
 898
 899         def __init__(self, youtube_ie, downloader=None):
 900                 InfoExtractor.__init__(self, downloader)
 901                 self._youtube_ie = youtube_ie
 902
 903         @staticmethod
 904         def suitable(url):
 905                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
 906
 907         def report_download_page(self, query, pagenum):
 908                 """Report attempt to download playlist page with given number."""
 909                 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
 910
 911         def _real_initialize(self):
 912                 self._youtube_ie.initialize()
 913
 914         def _real_extract(self, query):
 915                 mobj = re.match(self._VALID_QUERY, query)
 916                 if mobj is None:
 917                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
 918                         return
 919
 920                 prefix, query = query.split(':')
 921                 prefix = prefix[8:]
 922                 if prefix == '':
 923                         self._download_n_results(query, 1)
 924                         return
 925                 elif prefix == 'all':
 926                         self._download_n_results(query, self._max_youtube_results)
 927                         return
 928                 else:
 929                         try:
 930                                 n = long(prefix)
 931                                 if n <= 0:
 932                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
 933                                         return
 934                                 elif n > self._max_youtube_results:
 935                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
 936                                         n = self._max_youtube_results
 937                                 self._download_n_results(query, n)
 938                                 return
 939                         except ValueError: # parsing prefix as integer fails
 940                                 self._download_n_results(query, 1)
 941                                 return
 942
 943         def _download_n_results(self, query, n):
 944                 """Downloads a specified number of results for a query"""
 945
 946                 video_ids = []
 947                 already_seen = set()
 948                 pagenum = 1
 949
 950                 while True:
 951                         self.report_download_page(query, pagenum)
 952                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
 953                         request = urllib2.Request(result_url, None, std_headers)
 954                         try:
 955                                 page = urllib2.urlopen(request).read()
 956                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 957                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
 958                                 return
 959
 960                         # Extract video identifiers
 961                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
 962                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
 963                                 if video_id not in already_seen:
 964                                         video_ids.append(video_id)
 965                                         already_seen.add(video_id)
 966                                         if len(video_ids) == n:
 967                                                 # Specified n videos reached
 968                                                 for id in video_ids:
 969                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 970                                                 return
 971
 972                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
 973                                 for id in video_ids:
 974                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 975                                 return
 976
 977                         pagenum = pagenum + 1
 978
 979 class YoutubePlaylistIE(InfoExtractor):
 980         """Information Extractor for YouTube playlists."""
 981
 982         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:view_play_list|my_playlists)\?.*?p=([^&]+).*'
 983         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
 984         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
 985         _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
 986         _youtube_ie = None
 987
 988         def __init__(self, youtube_ie, downloader=None):
 989                 InfoExtractor.__init__(self, downloader)
 990                 self._youtube_ie = youtube_ie
 991
 992         @staticmethod
 993         def suitable(url):
 994                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
 995
 996         def report_download_page(self, playlist_id, pagenum):
 997                 """Report attempt to download playlist page with given number."""
 998                 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
 999
1000         def _real_initialize(self):
1001                 self._youtube_ie.initialize()
1002
1003         def _real_extract(self, url):
1004                 # Extract playlist id
1005                 mobj = re.match(self._VALID_URL, url)
1006                 if mobj is None:
1007                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1008                         return
1009
1010                 # Download playlist pages
1011                 playlist_id = mobj.group(1)
1012                 video_ids = []
1013                 pagenum = 1
1014
1015                 while True:
1016                         self.report_download_page(playlist_id, pagenum)
1017                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1018                         try:
1019                                 page = urllib2.urlopen(request).read()
1020                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1021                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1022                                 return
1023
1024                         # Extract video identifiers
1025                         ids_in_page = []
1026                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1027                                 if mobj.group(1) not in ids_in_page:
1028                                         ids_in_page.append(mobj.group(1))
1029                         video_ids.extend(ids_in_page)
1030
1031                         if (self._MORE_PAGES_INDICATOR % (playlist_id.upper(), pagenum + 1)) not in page:
1032                                 break
1033                         pagenum = pagenum + 1
1034
1035                 for id in video_ids:
1036                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1037                 return
1038
1039 class PostProcessor(object):
1040         """Post Processor class.
1041
1042         PostProcessor objects can be added to downloaders with their
1043         add_post_processor() method. When the downloader has finished a
1044         successful download, it will take its internal chain of PostProcessors
1045         and start calling the run() method on each one of them, first with
1046         an initial argument and then with the returned value of the previous
1047         PostProcessor.
1048
1049         The chain will be stopped if one of them ever returns None or the end
1050         of the chain is reached.
1051
1052         PostProcessor objects follow a "mutual registration" process similar
1053         to InfoExtractor objects.
1054         """
1055
1056         _downloader = None
1057
1058         def __init__(self, downloader=None):
1059                 self._downloader = downloader
1060
1061         def set_downloader(self, downloader):
1062                 """Sets the downloader for this PP."""
1063                 self._downloader = downloader
1064
1065         def run(self, information):
1066                 """Run the PostProcessor.
1067
1068                 The "information" argument is a dictionary like the ones
1069                 composed by InfoExtractors. The only difference is that this
1070                 one has an extra field called "filepath" that points to the
1071                 downloaded file.
1072
1073                 When this method returns None, the postprocessing chain is
1074                 stopped. However, this method may return an information
1075                 dictionary that will be passed to the next postprocessing
1076                 object in the chain. It can be the one it received after
1077                 changing some fields.
1078
1079                 In addition, this method may raise a PostProcessingError
1080                 exception that will be taken into account by the downloader
1081                 it was called from.
1082                 """
1083                 return information # by default, do nothing
1084
1085 ### MAIN PROGRAM ###
1086 if __name__ == '__main__':
1087         try:
1088                 # Modules needed only when running the main program
1089                 import getpass
1090                 import optparse
1091
1092                 # General configuration
1093                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1094                 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1095                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1096
1097                 # Parse command line
1098                 parser = optparse.OptionParser(
1099                         usage='Usage: %prog [options] url...',
1100                         version='2009.09.13',
1101                         conflict_handler='resolve',
1102                 )
1103
1104                 parser.add_option('-h', '--help',
1105                                 action='help', help='print this help text and exit')
1106                 parser.add_option('-v', '--version',
1107                                 action='version', help='print program version and exit')
1108                 parser.add_option('-i', '--ignore-errors',
1109                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1110                 parser.add_option('-r', '--rate-limit',
1111                                 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1112
1113                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1114                 authentication.add_option('-u', '--username',
1115                                 dest='username', metavar='UN', help='account username')
1116                 authentication.add_option('-p', '--password',
1117                                 dest='password', metavar='PW', help='account password')
1118                 authentication.add_option('-n', '--netrc',
1119                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1120                 parser.add_option_group(authentication)
1121
1122                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1123                 video_format.add_option('-f', '--format',
1124                                 action='store', dest='format', metavar='FMT', help='video format code')
1125                 video_format.add_option('-b', '--best-quality',
1126                                 action='store_const', dest='format', help='download the best quality video possible', const='0')
1127                 video_format.add_option('-m', '--mobile-version',
1128                                 action='store_const', dest='format', help='alias for -f 17', const='17')
1129                 video_format.add_option('-d', '--high-def',
1130                                 action='store_const', dest='format', help='alias for -f 22', const='22')
1131                 parser.add_option_group(video_format)
1132
1133                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1134                 verbosity.add_option('-q', '--quiet',
1135                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1136                 verbosity.add_option('-s', '--simulate',
1137                                 action='store_true', dest='simulate', help='do not download video', default=False)
1138                 verbosity.add_option('-g', '--get-url',
1139                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1140                 verbosity.add_option('-e', '--get-title',
1141                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1142                 parser.add_option_group(verbosity)
1143
1144                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1145                 filesystem.add_option('-t', '--title',
1146                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
1147                 filesystem.add_option('-l', '--literal',
1148                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1149                 filesystem.add_option('-o', '--output',
1150                                 dest='outtmpl', metavar='TPL', help='output filename template')
1151                 filesystem.add_option('-a', '--batch-file',
1152                                 dest='batchfile', metavar='F', help='file containing URLs to download')
1153                 filesystem.add_option('-w', '--no-overwrites',
1154                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1155                 filesystem.add_option('-c', '--continue',
1156                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1157                 parser.add_option_group(filesystem)
1158
1159                 (opts, args) = parser.parse_args()
1160
1161                 # Batch file verification
1162                 batchurls = []
1163                 if opts.batchfile is not None:
1164                         try:
1165                                 batchurls = open(opts.batchfile, 'r').readlines()
1166                                 batchurls = [x.strip() for x in batchurls]
1167                                 batchurls = [x for x in batchurls if len(x) > 0]
1168                         except IOError:
1169                                 sys.exit(u'ERROR: batch file could not be read')
1170                 all_urls = batchurls + args
1171
1172                 # Conflicting, missing and erroneous options
1173                 if len(all_urls) < 1:
1174                         parser.error(u'you must provide at least one URL')
1175                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1176                         parser.error(u'using .netrc conflicts with giving username/password')
1177                 if opts.password is not None and opts.username is None:
1178                         parser.error(u'account username missing')
1179                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1180                         parser.error(u'using output template conflicts with using title or literal title')
1181                 if opts.usetitle and opts.useliteral:
1182                         parser.error(u'using title conflicts with using literal title')
1183                 if opts.username is not None and opts.password is None:
1184                         opts.password = getpass.getpass(u'Type account password and press return:')
1185                 if opts.ratelimit is not None:
1186                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1187                         if numeric_limit is None:
1188                                 parser.error(u'invalid rate limit specified')
1189                         opts.ratelimit = numeric_limit
1190
1191                 # Information extractors
1192                 youtube_ie = YoutubeIE()
1193                 metacafe_ie = MetacafeIE(youtube_ie)
1194                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1195                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1196
1197                 # File downloader
1198                 fd = FileDownloader({
1199                         'usenetrc': opts.usenetrc,
1200                         'username': opts.username,
1201                         'password': opts.password,
1202                         'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1203                         'forceurl': opts.geturl,
1204                         'forcetitle': opts.gettitle,
1205                         'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1206                         'format': opts.format,
1207                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
1208                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1209                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1210                                 or u'%(id)s.%(ext)s'),
1211                         'ignoreerrors': opts.ignoreerrors,
1212                         'ratelimit': opts.ratelimit,
1213                         'nooverwrites': opts.nooverwrites,
1214                         'continuedl': opts.continue_dl,
1215                         })
1216                 fd.add_info_extractor(youtube_search_ie)
1217                 fd.add_info_extractor(youtube_pl_ie)
1218                 fd.add_info_extractor(metacafe_ie)
1219                 fd.add_info_extractor(youtube_ie)
1220                 retcode = fd.download(all_urls)
1221                 sys.exit(retcode)
1222
1223         except DownloadError:
1224                 sys.exit(1)
1225         except SameFileError:
1226                 sys.exit(u'ERROR: fixed output name but more than one file to download')
1227         except KeyboardInterrupt:
1228                 sys.exit(u'\nERROR: Interrupted by user')