X-Git-Url: https://git.rapsys.eu/youtubedl/blobdiff_plain/45ea099fd5d7f239294d8fff875ac509c45bea40..3c0d528426f11546b8125b6b2efa9640f85ed76e:/youtube-dl?ds=inline diff --git a/youtube-dl b/youtube-dl index 84b3f8e..4530c5d 100755 --- a/youtube-dl +++ b/youtube-dl @@ -1,250 +1,1676 @@ #!/usr/bin/env python -# -# Copyright (c) 2006 Ricardo Garcia Gonzalez -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# Except as contained in this notice, the name(s) of the above copyright -# holders shall not be used in advertising or otherwise to promote the -# sale, use or other dealings in this Software without prior written -# authorization. -# -import sys -import optparse +# -*- coding: utf-8 -*- +# Author: Ricardo Garcia Gonzalez +# Author: Danny Colligan +# Author: Benjamin Johnson +# License: Public domain code +import htmlentitydefs import httplib -import urllib2 +import locale +import math +import netrc +import os +import os.path import re +import socket +import string +import subprocess +import sys +import time +import urllib +import urllib2 -# First off, check Python and refuse to run -if sys.hexversion < 0x020400f0: - sys.exit('Error: Python 2.4 or later needed to run the program') - -# Exit status constants -exit_failure = 1 -exit_success = 0 - -# Global constants -const_video_url_str = 'http://www.youtube.com/watch?v=%s' -const_video_url_re = re.compile(r'http://(?:www\.)?youtube\.com/(?:v/|(?:watch(?:\.php)?)?\?v=)([^&]+).*') -const_login_url_str = 'http://www.youtube.com/signup?next=/' -const_login_post_str = 'current_form=loginForm&next=%%2F&username=%s&password=%s&action_login=Log+In' -const_age_url_str = 'http://www.youtube.com/verify_age?next_url=/watch%%3Fv%%3D%s' -const_age_post_str = 'next_url=%%2Fwatch%%3Fv%%3D%s&action_confirm=Confirm' -const_video_url_params_re = re.compile(r'player2\.swf\?([^"]+)"', re.M) -const_video_url_real_str = 'http://www.youtube.com/get_video?%s' -const_1k = 1024 -const_block_size = 10 * const_1k - -# Print error message, followed by standard advice information, and then exit -def error_advice_exit(error_text): - global exit_failure - sys.stderr.write('Error: %s.\n' % error_text) - sys.stderr.write('Try again several times. It may be a temporal problem.\n') - sys.stderr.write('Other typical problems:\n\n') - sys.stderr.write('\tVideo no longer exists.\n') - sys.stderr.write('\tVideo requires age confirmation but you did not provide an account.\n') - sys.stderr.write('\tYou provided the account data, but it is not valid.\n') - sys.stderr.write('\tThe connection was cut suddenly for some reason.\n') - sys.stderr.write('\tYouTube changed their system, and the program no longer works.\n') - sys.stderr.write('\nTry to confirm you are able to view the video using a web browser.\n') - sys.stderr.write('Use the same video URL and account information, if needed, with this program.\n') - sys.stderr.write('When using a proxy, make sure http_proxy has http://host:port format.\n') - sys.stderr.write('Try again several times and contact me if the problem persists.\n') - sys.exit(exit_failure) - -# Wrapper to create custom requests with typical headers -def request_create(url, data=None): - retval = urllib2.Request(url) - if not data is None: - retval.add_data(data) - # Try to mimic Firefox, at least a little bit - retval.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.6) Gecko/20060728 Firefox/1.5.0.6') - retval.add_header('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7') - retval.add_header('Accept', 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5') - retval.add_header('Accept-Language', 'en-us,en;q=0.5') - return retval - -# Perform a request, process headers and return response -def perform_request(url, data=None): - request = request_create(url, data) - response = urllib2.urlopen(request) - return response - -# Convert bytes to KiB -def to_k(bytes): - global const_1k - return bytes / const_1k - -# Conditional print -def cond_print(str): - global cmdl_opts - if not cmdl_opts.quiet: - sys.stdout.write(str) - sys.stdout.flush() - -# Create the command line options parser and parse command line -cmdl_usage = 'usage: %prog [options] video_url' -cmdl_version = '2006.09.25' -cmdl_parser = optparse.OptionParser(usage=cmdl_usage, version=cmdl_version, conflict_handler='resolve') -cmdl_parser.add_option('-h', '--help', action='help', help='print this help text and exit') -cmdl_parser.add_option('-v', '--version', action='version', help='print program version and exit') -cmdl_parser.add_option('-u', '--username', dest='username', metavar='USERNAME', help='account username') -cmdl_parser.add_option('-p', '--password', dest='password', metavar='PASSWORD', help='account password') -cmdl_parser.add_option('-o', '--output', dest='outfile', metavar='FILE', help='output video file name') -cmdl_parser.add_option('-q', '--quiet', action='store_true', dest='quiet', help='activates quiet mode') -cmdl_parser.add_option('-s', '--simulate', action='store_true', dest='simulate', help='do not download video') -(cmdl_opts, cmdl_args) = cmdl_parser.parse_args() - -# Get video URL -if len(cmdl_args) != 1: - cmdl_parser.print_help() - sys.exit(exit_failure) -video_url_cmdl = cmdl_args[0] - -# Verify video URL format and convert to "standard" format -video_url_mo = const_video_url_re.match(video_url_cmdl) -if video_url_mo is None: - sys.exit('Error: URL does not seem to be a youtube video URL. If it is, report a bug.') -video_url_id = video_url_mo.group(1) -video_url = const_video_url_str % video_url_id - -# Check conflicting options -if not cmdl_opts.outfile is None and cmdl_opts.simulate: - sys.stderr.write('Warning: video file name given but will not be used.\n') - -# Get output file name -if cmdl_opts.outfile is None: - video_filename = '%s.flv' % video_url_id -else: - video_filename = cmdl_opts.outfile - -# Check name -if not video_filename.lower().endswith('.flv'): - sys.stderr.write('Warning: video file name does not end in .flv\n') - -# Verify both or none present -if ((cmdl_opts.username is None and not cmdl_opts.password is None) or - (not cmdl_opts.username is None and cmdl_opts.password is None)): - sys.exit('Error: both username and password must be given, or none.') - -# Test writable file -if not cmdl_opts.simulate: - try: - disk_test = open(video_filename, 'wb') - disk_test.close() +# parse_qs was moved from the cgi module to the urlparse module recently. +try: + from urlparse import parse_qs +except ImportError: + from cgi import parse_qs + +std_headers = { + 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6', + 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', + 'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', + 'Accept-Language': 'en-us,en;q=0.5', +} - except (OSError, IOError): - sys.exit('Error: unable to open %s for writing.' % video_filename) +simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii') -# Install cookie and proxy handlers -urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor())) -urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler())) +def preferredencoding(): + """Get preferred encoding. -# Login and confirm age if needed -if not cmdl_opts.username is None: + Returns the best encoding scheme for the system, based on + locale.getpreferredencoding() and some further tweaks. + """ + def yield_preferredencoding(): + try: + pref = locale.getpreferredencoding() + u'TEST'.encode(pref) + except: + pref = 'UTF-8' + while True: + yield pref + return yield_preferredencoding().next() + +def htmlentity_transform(matchobj): + """Transforms an HTML entity to a Unicode character. + + This function receives a match object and is intended to be used with + the re.sub() function. + """ + entity = matchobj.group(1) + + # Known non-numeric HTML entity + if entity in htmlentitydefs.name2codepoint: + return unichr(htmlentitydefs.name2codepoint[entity]) + + # Unicode character + mobj = re.match(ur'(?u)#(x?\d+)', entity) + if mobj is not None: + numstr = mobj.group(1) + if numstr.startswith(u'x'): + base = 16 + numstr = u'0%s' % numstr + else: + base = 10 + return unichr(long(numstr, base)) + + # Unknown entity in name, return its literal representation + return (u'&%s;' % entity) + +def sanitize_title(utitle): + """Sanitizes a video title so it could be used as part of a filename.""" + utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle) + return utitle.replace(unicode(os.sep), u'%') + +def sanitize_open(filename, open_mode): + """Try to open the given filename, and slightly tweak it if this fails. + + Attempts to open the given filename. If this fails, it tries to change + the filename slightly, step by step, until it's either able to open it + or it fails and raises a final exception, like the standard open() + function. + + It returns the tuple (stream, definitive_file_name). + """ try: + stream = open(filename, open_mode) + return (stream, filename) + except (IOError, OSError), err: + # In case of error, try to remove win32 forbidden chars + filename = re.sub(ur'[<>:"\|\?\*]', u'#', filename) + + # An exception here should be caught in the caller + stream = open(filename, open_mode) + return (stream, filename) + + +class DownloadError(Exception): + """Download Error exception. + + This exception may be thrown by FileDownloader objects if they are not + configured to continue on errors. They will contain the appropriate + error message. + """ + pass + +class SameFileError(Exception): + """Same File exception. + + This exception will be thrown by FileDownloader objects if they detect + multiple files would have to be downloaded to the same file on disk. + """ + pass + +class PostProcessingError(Exception): + """Post Processing exception. + + This exception may be raised by PostProcessor's .run() method to + indicate an error in the postprocessing task. + """ + pass + +class UnavailableFormatError(Exception): + """Unavailable Format exception. + + This exception will be thrown when a video is requested + in a format that is not available for that video. + """ + pass + +class ContentTooShortError(Exception): + """Content Too Short exception. + + This exception may be raised by FileDownloader objects when a file they + download is too small for what the server announced first, indicating + the connection was probably interrupted. + """ + # Both in bytes + downloaded = None + expected = None + + def __init__(self, downloaded, expected): + self.downloaded = downloaded + self.expected = expected + +class FileDownloader(object): + """File Downloader class. + + File downloader objects are the ones responsible of downloading the + actual video file and writing it to disk if the user has requested + it, among some other tasks. In most cases there should be one per + program. As, given a video URL, the downloader doesn't know how to + extract all the needed information, task that InfoExtractors do, it + has to pass the URL to one of them. + + For this, file downloader objects have a method that allows + InfoExtractors to be registered in a given order. When it is passed + a URL, the file downloader handles it to the first InfoExtractor it + finds that reports being able to handle it. The InfoExtractor extracts + all the information about the video or videos the URL refers to, and + asks the FileDownloader to process the video information, possibly + downloading the video. + + File downloaders accept a lot of parameters. In order not to saturate + the object constructor with arguments, it receives a dictionary of + options instead. These options are available through the params + attribute for the InfoExtractors to use. The FileDownloader also + registers itself as the downloader in charge for the InfoExtractors + that are added to it, so this is a "mutual registration". + + Available options: + + username: Username for authentication purposes. + password: Password for authentication purposes. + usenetrc: Use netrc for authentication instead. + quiet: Do not print messages to stdout. + forceurl: Force printing final URL. + forcetitle: Force printing title. + simulate: Do not download the video files. + format: Video format code. + outtmpl: Template for output names. + ignoreerrors: Do not stop on download errors. + ratelimit: Download speed limit, in bytes/sec. + nooverwrites: Prevent overwriting files. + continuedl: Try to continue downloads if possible. + noprogress: Do not print the progress bar. + """ + + params = None + _ies = [] + _pps = [] + _download_retcode = None + + def __init__(self, params): + """Create a FileDownloader object with the given options.""" + self._ies = [] + self._pps = [] + self._download_retcode = 0 + self.params = params + + @staticmethod + def pmkdir(filename): + """Create directory components in filename. Similar to Unix "mkdir -p".""" + components = filename.split(os.sep) + aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))] + aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator + for dir in aggregate: + if not os.path.exists(dir): + os.mkdir(dir) + + @staticmethod + def format_bytes(bytes): + if bytes is None: + return 'N/A' + if type(bytes) is str: + bytes = float(bytes) + if bytes == 0.0: + exponent = 0 + else: + exponent = long(math.log(bytes, 1024.0)) + suffix = 'bkMGTPEZY'[exponent] + converted = float(bytes) / float(1024**exponent) + return '%.2f%s' % (converted, suffix) + + @staticmethod + def calc_percent(byte_counter, data_len): + if data_len is None: + return '---.-%' + return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0)) + + @staticmethod + def calc_eta(start, now, total, current): + if total is None: + return '--:--' + dif = now - start + if current == 0 or dif < 0.001: # One millisecond + return '--:--' + rate = float(current) / dif + eta = long((float(total) - float(current)) / rate) + (eta_mins, eta_secs) = divmod(eta, 60) + if eta_mins > 99: + return '--:--' + return '%02d:%02d' % (eta_mins, eta_secs) + + @staticmethod + def calc_speed(start, now, bytes): + dif = now - start + if bytes == 0 or dif < 0.001: # One millisecond + return '%10s' % '---b/s' + return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif)) + + @staticmethod + def best_block_size(elapsed_time, bytes): + new_min = max(bytes / 2.0, 1.0) + new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB + if elapsed_time < 0.001: + return long(new_max) + rate = bytes / elapsed_time + if rate > new_max: + return long(new_max) + if rate < new_min: + return long(new_min) + return long(rate) + + @staticmethod + def parse_bytes(bytestr): + """Parse a string indicating a byte quantity into a long integer.""" + matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) + if matchobj is None: + return None + number = float(matchobj.group(1)) + multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) + return long(round(number * multiplier)) + + @staticmethod + def verify_url(url): + """Verify a URL is valid and data could be downloaded. Return real data URL.""" + request = urllib2.Request(url, None, std_headers) + data = urllib2.urlopen(request) + data.read(1) + url = data.geturl() + data.close() + return url + + def add_info_extractor(self, ie): + """Add an InfoExtractor object to the end of the list.""" + self._ies.append(ie) + ie.set_downloader(self) + + def add_post_processor(self, pp): + """Add a PostProcessor object to the end of the chain.""" + self._pps.append(pp) + pp.set_downloader(self) + + def to_stdout(self, message, skip_eol=False, ignore_encoding_errors=False): + """Print message to stdout if not in quiet mode.""" + try: + if not self.params.get('quiet', False): + print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()), + sys.stdout.flush() + except (UnicodeEncodeError), err: + if not ignore_encoding_errors: + raise + + def to_stderr(self, message): + """Print message to stderr.""" + print >>sys.stderr, message.encode(preferredencoding()) + + def fixed_template(self): + """Checks if the output template is fixed.""" + return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None) + + def trouble(self, message=None): + """Determine action to take when a download problem appears. + + Depending on if the downloader has been configured to ignore + download errors or not, this method may throw an exception or + not when errors are found, after printing the message. + """ + if message is not None: + self.to_stderr(message) + if not self.params.get('ignoreerrors', False): + raise DownloadError(message) + self._download_retcode = 1 + + def slow_down(self, start_time, byte_counter): + """Sleep if the download speed is over the rate limit.""" + rate_limit = self.params.get('ratelimit', None) + if rate_limit is None or byte_counter == 0: + return + now = time.time() + elapsed = now - start_time + if elapsed <= 0.0: + return + speed = float(byte_counter) / elapsed + if speed > rate_limit: + time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) + + def report_destination(self, filename): + """Report destination filename.""" + self.to_stdout(u'[download] Destination: %s' % filename, ignore_encoding_errors=True) + + def report_progress(self, percent_str, data_len_str, speed_str, eta_str): + """Report download progress.""" + if self.params.get('noprogress', False): + return + self.to_stdout(u'\r[download] %s of %s at %s ETA %s' % + (percent_str, data_len_str, speed_str, eta_str), skip_eol=True) + + def report_resuming_byte(self, resume_len): + """Report attemtp to resume at given byte.""" + self.to_stdout(u'[download] Resuming download at byte %s' % resume_len) + + def report_file_already_downloaded(self, file_name): + """Report file has already been fully downloaded.""" + try: + self.to_stdout(u'[download] %s has already been downloaded' % file_name) + except (UnicodeEncodeError), err: + self.to_stdout(u'[download] The file has already been downloaded') + + def report_unable_to_resume(self): + """Report it was impossible to resume download.""" + self.to_stdout(u'[download] Unable to resume') + + def report_finish(self): + """Report download finished.""" + if self.params.get('noprogress', False): + self.to_stdout(u'[download] Download completed') + else: + self.to_stdout(u'') + + def process_info(self, info_dict): + """Process a single dictionary returned by an InfoExtractor.""" + # Do nothing else if in simulate mode + if self.params.get('simulate', False): + # Verify URL if it's an HTTP one + if info_dict['url'].startswith('http'): + try: + self.verify_url(info_dict['url'].encode('utf-8')).decode('utf-8') + except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err: + raise UnavailableFormatError + + # Forced printings + if self.params.get('forcetitle', False): + print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace') + if self.params.get('forceurl', False): + print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace') + + return + + try: + template_dict = dict(info_dict) + template_dict['epoch'] = unicode(long(time.time())) + filename = self.params['outtmpl'] % template_dict + except (ValueError, KeyError), err: + self.trouble('ERROR: invalid output template or system charset: %s' % str(err)) + if self.params.get('nooverwrites', False) and os.path.exists(filename): + self.to_stderr(u'WARNING: file exists: %s; skipping' % filename) + return + + try: + self.pmkdir(filename) + except (OSError, IOError), err: + self.trouble('ERROR: unable to create directories: %s' % str(err)) + return + + try: + success = self._do_download(filename, info_dict['url'].encode('utf-8')) + except (OSError, IOError), err: + raise UnavailableFormatError + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self.trouble('ERROR: unable to download video data: %s' % str(err)) + return + except (ContentTooShortError, ), err: + self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) + return + + if success: + try: + self.post_process(filename, info_dict) + except (PostProcessingError), err: + self.trouble('ERROR: postprocessing: %s' % str(err)) + return + + def download(self, url_list): + """Download a given list of URLs.""" + if len(url_list) > 1 and self.fixed_template(): + raise SameFileError(self.params['outtmpl']) + + for url in url_list: + suitable_found = False + for ie in self._ies: + # Go to next InfoExtractor if not suitable + if not ie.suitable(url): + continue + + # Suitable InfoExtractor found + suitable_found = True + + # Extract information from URL and process it + ie.extract(url) + + # Suitable InfoExtractor had been found; go to next URL + break + + if not suitable_found: + self.trouble('ERROR: no suitable InfoExtractor: %s' % url) + + return self._download_retcode + + def post_process(self, filename, ie_info): + """Run the postprocessing chain on the given file.""" + info = dict(ie_info) + info['filepath'] = filename + for pp in self._pps: + info = pp.run(info) + if info is None: + break + + def _download_with_rtmpdump(self, filename, url): + self.report_destination(filename) + + # Check for rtmpdump first + try: + subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT) + except (OSError, IOError): + self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run') + return False + + # Download using rtmpdump. rtmpdump returns exit code 2 when + # the connection was interrumpted and resuming appears to be + # possible. This is part of rtmpdump's normal usage, AFAIK. + basic_args = ['rtmpdump', '-q', '-r', url, '-o', filename] + retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]) + while retval == 2 or retval == 1: + self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename), skip_eol=True) + time.sleep(2.0) # This seems to be needed + retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) + if retval == 0: + self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename)) + return True + else: + self.trouble('ERROR: rtmpdump exited with code %d' % retval) + return False + + def _do_download(self, filename, url): + # Attempt to download using rtmpdump + if url.startswith('rtmp'): + return self._download_with_rtmpdump(filename, url) + + stream = None + open_mode = 'wb' + basic_request = urllib2.Request(url, None, std_headers) + request = urllib2.Request(url, None, std_headers) + + # Establish possible resume length + if os.path.isfile(filename): + resume_len = os.path.getsize(filename) + else: + resume_len = 0 + + # Request parameters in case of being able to resume + if self.params.get('continuedl', False) and resume_len != 0: + self.report_resuming_byte(resume_len) + request.add_header('Range','bytes=%d-' % resume_len) + open_mode = 'ab' + + # Establish connection + try: + data = urllib2.urlopen(request) + except (urllib2.HTTPError, ), err: + if err.code != 416: # 416 is 'Requested range not satisfiable' + raise + # Unable to resume + data = urllib2.urlopen(basic_request) + content_length = data.info()['Content-Length'] + + if content_length is not None and long(content_length) == resume_len: + # Because the file had already been fully downloaded + self.report_file_already_downloaded(filename) + return True + else: + # Because the server didn't let us + self.report_unable_to_resume() + open_mode = 'wb' + + data_len = data.info().get('Content-length', None) + data_len_str = self.format_bytes(data_len) + byte_counter = 0 + block_size = 1024 + start = time.time() + while True: + # Download and write + before = time.time() + data_block = data.read(block_size) + after = time.time() + data_block_len = len(data_block) + if data_block_len == 0: + break + byte_counter += data_block_len + + # Open file just in time + if stream is None: + try: + (stream, filename) = sanitize_open(filename, open_mode) + self.report_destination(filename) + except (OSError, IOError), err: + self.trouble('ERROR: unable to open for writing: %s' % str(err)) + return False + stream.write(data_block) + block_size = self.best_block_size(after - before, data_block_len) + + # Progress message + percent_str = self.calc_percent(byte_counter, data_len) + eta_str = self.calc_eta(start, time.time(), data_len, byte_counter) + speed_str = self.calc_speed(start, time.time(), byte_counter) + self.report_progress(percent_str, data_len_str, speed_str, eta_str) + + # Apply rate limit + self.slow_down(start, byte_counter) + + self.report_finish() + if data_len is not None and str(byte_counter) != data_len: + raise ContentTooShortError(byte_counter, long(data_len)) + return True + +class InfoExtractor(object): + """Information Extractor class. + + Information extractors are the classes that, given a URL, extract + information from the video (or videos) the URL refers to. This + information includes the real video URL, the video title and simplified + title, author and others. The information is stored in a dictionary + which is then passed to the FileDownloader. The FileDownloader + processes this information possibly downloading the video to the file + system, among other possible outcomes. The dictionaries must include + the following fields: + + id: Video identifier. + url: Final video URL. + uploader: Nickname of the video uploader. + title: Literal title. + stitle: Simplified title. + ext: Video filename extension. + + Subclasses of this one should re-define the _real_initialize() and + _real_extract() methods, as well as the suitable() static method. + Probably, they should also be instantiated and added to the main + downloader. + """ + + _ready = False + _downloader = None + + def __init__(self, downloader=None): + """Constructor. Receives an optional downloader.""" + self._ready = False + self.set_downloader(downloader) + + @staticmethod + def suitable(url): + """Receives a URL and returns True if suitable for this IE.""" + return False + + def initialize(self): + """Initializes an instance (authentication, etc).""" + if not self._ready: + self._real_initialize() + self._ready = True + + def extract(self, url): + """Extracts URL information and returns it in list of dicts.""" + self.initialize() + return self._real_extract(url) + + def set_downloader(self, downloader): + """Sets the downloader for this IE.""" + self._downloader = downloader + + def _real_initialize(self): + """Real initialization process. Redefine in subclasses.""" + pass + + def _real_extract(self, url): + """Real extraction process. Redefine in subclasses.""" + pass + +class YoutubeIE(InfoExtractor): + """Information extractor for youtube.com.""" + + _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?[\?#](?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$' + _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' + _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en' + _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' + _NETRC_MACHINE = 'youtube' + _available_formats = ['37', '22', '35', '18', '34', '5', '17', '13', None] # listed in order of priority for -b flag + _video_extensions = { + '13': '3gp', + '17': 'mp4', + '18': 'mp4', + '22': 'mp4', + '37': 'mp4', + } + + @staticmethod + def suitable(url): + return (re.match(YoutubeIE._VALID_URL, url) is not None) + + def report_lang(self): + """Report attempt to set language.""" + self._downloader.to_stdout(u'[youtube] Setting language') + + def report_login(self): + """Report attempt to log in.""" + self._downloader.to_stdout(u'[youtube] Logging in') + + def report_age_confirmation(self): + """Report attempt to confirm age.""" + self._downloader.to_stdout(u'[youtube] Confirming age') + + def report_video_info_webpage_download(self, video_id): + """Report attempt to download video info webpage.""" + self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id) + + def report_information_extraction(self, video_id): + """Report attempt to extract video information.""" + self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id) + + def report_unavailable_format(self, video_id, format): + """Report extracted video URL.""" + self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format)) + + def report_rtmp_download(self): + """Indicate the download will use the RTMP protocol.""" + self._downloader.to_stdout(u'[youtube] RTMP download detected') + + def _real_initialize(self): + if self._downloader is None: + return + + username = None + password = None + downloader_params = self._downloader.params + + # Attempt to use provided username and password or .netrc data + if downloader_params.get('username', None) is not None: + username = downloader_params['username'] + password = downloader_params['password'] + elif downloader_params.get('usenetrc', False): + try: + info = netrc.netrc().authenticators(self._NETRC_MACHINE) + if info is not None: + username = info[0] + password = info[2] + else: + raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) + except (IOError, netrc.NetrcParseError), err: + self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err)) + return + + # Set language + request = urllib2.Request(self._LANG_URL, None, std_headers) + try: + self.report_lang() + urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err)) + return + + # No authentication to be performed + if username is None: + return + # Log in - cond_print('Logging in... ') - perform_request(const_login_url_str, const_login_post_str % (cmdl_opts.username, cmdl_opts.password)).read() - cond_print('done.\n') + login_form = { + 'current_form': 'loginForm', + 'next': '/', + 'action_login': 'Log In', + 'username': username, + 'password': password, + } + request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers) + try: + self.report_login() + login_results = urllib2.urlopen(request).read() + if re.search(r'(?i)