+ stream = open(filename, open_mode)
+ return (stream, filename)
+ except (IOError, OSError), err:
+ # In case of error, try to remove win32 forbidden chars
+ filename = re.sub(ur'[<>:"\|\?\*]', u'#', filename)
+
+ # An exception here should be caught in the caller
+ stream = open(filename, open_mode)
+ return (stream, filename)
+
+
+class DownloadError(Exception):
+ """Download Error exception.
+
+ This exception may be thrown by FileDownloader objects if they are not
+ configured to continue on errors. They will contain the appropriate
+ error message.
+ """
+ pass
+
+class SameFileError(Exception):
+ """Same File exception.
+
+ This exception will be thrown by FileDownloader objects if they detect
+ multiple files would have to be downloaded to the same file on disk.
+ """
+ pass
+
+class PostProcessingError(Exception):
+ """Post Processing exception.
+
+ This exception may be raised by PostProcessor's .run() method to
+ indicate an error in the postprocessing task.
+ """
+ pass
+
+class UnavailableFormatError(Exception):
+ """Unavailable Format exception.
+
+ This exception will be thrown when a video is requested
+ in a format that is not available for that video.
+ """
+ pass
+
+class ContentTooShortError(Exception):
+ """Content Too Short exception.
+
+ This exception may be raised by FileDownloader objects when a file they
+ download is too small for what the server announced first, indicating
+ the connection was probably interrupted.
+ """
+ # Both in bytes
+ downloaded = None
+ expected = None
+
+ def __init__(self, downloaded, expected):
+ self.downloaded = downloaded
+ self.expected = expected
+
+class FileDownloader(object):
+ """File Downloader class.
+
+ File downloader objects are the ones responsible of downloading the
+ actual video file and writing it to disk if the user has requested
+ it, among some other tasks. In most cases there should be one per
+ program. As, given a video URL, the downloader doesn't know how to
+ extract all the needed information, task that InfoExtractors do, it
+ has to pass the URL to one of them.
+
+ For this, file downloader objects have a method that allows
+ InfoExtractors to be registered in a given order. When it is passed
+ a URL, the file downloader handles it to the first InfoExtractor it
+ finds that reports being able to handle it. The InfoExtractor extracts
+ all the information about the video or videos the URL refers to, and
+ asks the FileDownloader to process the video information, possibly
+ downloading the video.
+
+ File downloaders accept a lot of parameters. In order not to saturate
+ the object constructor with arguments, it receives a dictionary of
+ options instead. These options are available through the params
+ attribute for the InfoExtractors to use. The FileDownloader also
+ registers itself as the downloader in charge for the InfoExtractors
+ that are added to it, so this is a "mutual registration".
+
+ Available options:
+
+ username: Username for authentication purposes.
+ password: Password for authentication purposes.
+ usenetrc: Use netrc for authentication instead.
+ quiet: Do not print messages to stdout.
+ forceurl: Force printing final URL.
+ forcetitle: Force printing title.
+ simulate: Do not download the video files.
+ format: Video format code.
+ outtmpl: Template for output names.
+ ignoreerrors: Do not stop on download errors.
+ ratelimit: Download speed limit, in bytes/sec.
+ nooverwrites: Prevent overwriting files.
+ continuedl: Try to continue downloads if possible.
+ """
+
+ params = None
+ _ies = []
+ _pps = []
+ _download_retcode = None
+
+ def __init__(self, params):
+ """Create a FileDownloader object with the given options."""
+ self._ies = []
+ self._pps = []
+ self._download_retcode = 0
+ self.params = params
+
+ @staticmethod
+ def pmkdir(filename):
+ """Create directory components in filename. Similar to Unix "mkdir -p"."""
+ components = filename.split(os.sep)
+ aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
+ aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
+ for dir in aggregate:
+ if not os.path.exists(dir):
+ os.mkdir(dir)
+
+ @staticmethod
+ def format_bytes(bytes):
+ if bytes is None:
+ return 'N/A'
+ if type(bytes) is str:
+ bytes = float(bytes)
+ if bytes == 0.0:
+ exponent = 0
+ else:
+ exponent = long(math.log(bytes, 1024.0))
+ suffix = 'bkMGTPEZY'[exponent]
+ converted = float(bytes) / float(1024**exponent)
+ return '%.2f%s' % (converted, suffix)
+
+ @staticmethod
+ def calc_percent(byte_counter, data_len):
+ if data_len is None:
+ return '---.-%'
+ return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
+
+ @staticmethod
+ def calc_eta(start, now, total, current):
+ if total is None:
+ return '--:--'
+ dif = now - start
+ if current == 0 or dif < 0.001: # One millisecond
+ return '--:--'
+ rate = float(current) / dif
+ eta = long((float(total) - float(current)) / rate)
+ (eta_mins, eta_secs) = divmod(eta, 60)
+ if eta_mins > 99:
+ return '--:--'
+ return '%02d:%02d' % (eta_mins, eta_secs)
+
+ @staticmethod
+ def calc_speed(start, now, bytes):
+ dif = now - start
+ if bytes == 0 or dif < 0.001: # One millisecond
+ return '%10s' % '---b/s'
+ return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
+
+ @staticmethod
+ def best_block_size(elapsed_time, bytes):
+ new_min = max(bytes / 2.0, 1.0)
+ new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
+ if elapsed_time < 0.001:
+ return long(new_max)
+ rate = bytes / elapsed_time
+ if rate > new_max:
+ return long(new_max)
+ if rate < new_min:
+ return long(new_min)
+ return long(rate)
+
+ @staticmethod
+ def parse_bytes(bytestr):
+ """Parse a string indicating a byte quantity into a long integer."""
+ matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
+ if matchobj is None:
+ return None
+ number = float(matchobj.group(1))
+ multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
+ return long(round(number * multiplier))
+
+ @staticmethod
+ def verify_url(url):
+ """Verify a URL is valid and data could be downloaded. Return real data URL."""
+ request = urllib2.Request(url, None, std_headers)
+ data = urllib2.urlopen(request)
+ data.read(1)
+ url = data.geturl()
+ data.close()
+ return url
+
+ def add_info_extractor(self, ie):
+ """Add an InfoExtractor object to the end of the list."""
+ self._ies.append(ie)
+ ie.set_downloader(self)
+
+ def add_post_processor(self, pp):
+ """Add a PostProcessor object to the end of the chain."""
+ self._pps.append(pp)
+ pp.set_downloader(self)
+
+ def to_stdout(self, message, skip_eol=False):
+ """Print message to stdout if not in quiet mode."""
+ if not self.params.get('quiet', False):
+ print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
+ sys.stdout.flush()
+
+ def to_stderr(self, message):
+ """Print message to stderr."""
+ print >>sys.stderr, message.encode(preferredencoding())
+
+ def fixed_template(self):
+ """Checks if the output template is fixed."""
+ return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
+
+ def trouble(self, message=None):
+ """Determine action to take when a download problem appears.
+
+ Depending on if the downloader has been configured to ignore
+ download errors or not, this method may throw an exception or
+ not when errors are found, after printing the message.
+ """
+ if message is not None:
+ self.to_stderr(message)
+ if not self.params.get('ignoreerrors', False):
+ raise DownloadError(message)
+ self._download_retcode = 1
+
+ def slow_down(self, start_time, byte_counter):
+ """Sleep if the download speed is over the rate limit."""
+ rate_limit = self.params.get('ratelimit', None)
+ if rate_limit is None or byte_counter == 0:
+ return
+ now = time.time()
+ elapsed = now - start_time
+ if elapsed <= 0.0:
+ return
+ speed = float(byte_counter) / elapsed
+ if speed > rate_limit:
+ time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
+
+ def report_destination(self, filename):
+ """Report destination filename."""
+ self.to_stdout(u'[download] Destination: %s' % filename)
+
+ def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
+ """Report download progress."""
+ self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
+ (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
+
+ def report_resuming_byte(self, resume_len):
+ """Report attemtp to resume at given byte."""
+ self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
+
+ def report_file_already_downloaded(self, file_name):
+ """Report file has already been fully downloaded."""
+ self.to_stdout(u'[download] %s has already been downloaded' % file_name)
+
+ def report_unable_to_resume(self):
+ """Report it was impossible to resume download."""
+ self.to_stdout(u'[download] Unable to resume')
+
+ def report_finish(self):
+ """Report download finished."""
+ self.to_stdout(u'')
+
+ def process_info(self, info_dict):
+ """Process a single dictionary returned by an InfoExtractor."""
+ # Do nothing else if in simulate mode
+ if self.params.get('simulate', False):
+ # Verify URL if it's an HTTP one
+ if info_dict['url'].startswith('http'):
+ try:
+ info_dict['url'] = self.verify_url(info_dict['url'].encode('utf-8')).decode('utf-8')
+ except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
+ raise UnavailableFormatError
+
+ # Forced printings
+ if self.params.get('forcetitle', False):
+ print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
+ if self.params.get('forceurl', False):
+ print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
+
+ return
+
+ try:
+ template_dict = dict(info_dict)
+ template_dict['epoch'] = unicode(long(time.time()))
+ filename = self.params['outtmpl'] % template_dict
+ except (ValueError, KeyError), err:
+ self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
+ if self.params.get('nooverwrites', False) and os.path.exists(filename):
+ self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
+ return
+
+ try:
+ self.pmkdir(filename)
+ except (OSError, IOError), err:
+ self.trouble('ERROR: unable to create directories: %s' % str(err))
+ return
+
+ try:
+ success = self._do_download(filename, info_dict['url'].encode('utf-8'))
+ except (OSError, IOError), err:
+ raise UnavailableFormatError
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self.trouble('ERROR: unable to download video data: %s' % str(err))
+ return
+ except (ContentTooShortError, ), err:
+ self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+ return
+
+ if success:
+ try:
+ self.post_process(filename, info_dict)
+ except (PostProcessingError), err:
+ self.trouble('ERROR: postprocessing: %s' % str(err))
+ return
+
+ def download(self, url_list):
+ """Download a given list of URLs."""
+ if len(url_list) > 1 and self.fixed_template():
+ raise SameFileError(self.params['outtmpl'])
+
+ for url in url_list:
+ suitable_found = False
+ for ie in self._ies:
+ # Go to next InfoExtractor if not suitable
+ if not ie.suitable(url):
+ continue
+
+ # Suitable InfoExtractor found
+ suitable_found = True
+
+ # Extract information from URL and process it
+ ie.extract(url)
+
+ # Suitable InfoExtractor had been found; go to next URL
+ break
+
+ if not suitable_found:
+ self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
+
+ return self._download_retcode
+
+ def post_process(self, filename, ie_info):
+ """Run the postprocessing chain on the given file."""
+ info = dict(ie_info)
+ info['filepath'] = filename
+ for pp in self._pps:
+ info = pp.run(info)
+ if info is None:
+ break
+
+ def _download_with_rtmpdump(self, filename, url):
+ self.report_destination(filename)
+
+ # Check for rtmpdump first
+ try:
+ subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+ except (OSError, IOError):
+ self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
+ return False
+
+ # Download using rtmpdump. rtmpdump returns exit code 2 when
+ # the connection was interrumpted and resuming appears to be
+ # possible. This is part of rtmpdump's normal usage, AFAIK.
+ basic_args = ['rtmpdump', '-q', '-r', url, '-o', filename]
+ retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
+ while retval == 2 or retval == 1:
+ self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename), skip_eol=True)
+ time.sleep(2.0) # This seems to be needed
+ retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
+ if retval == 0:
+ self.to_stdout(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
+ return True
+ else:
+ self.trouble('ERROR: rtmpdump exited with code %d' % retval)
+ return False
+
+ def _do_download(self, filename, url):
+ # Attempt to download using rtmpdump
+ if url.startswith('rtmp'):
+ return self._download_with_rtmpdump(filename, url)
+
+ stream = None
+ open_mode = 'wb'
+ basic_request = urllib2.Request(url, None, std_headers)
+ request = urllib2.Request(url, None, std_headers)
+
+ # Establish possible resume length
+ if os.path.isfile(filename):
+ resume_len = os.path.getsize(filename)
+ else:
+ resume_len = 0
+
+ # Request parameters in case of being able to resume
+ if self.params.get('continuedl', False) and resume_len != 0:
+ self.report_resuming_byte(resume_len)
+ request.add_header('Range','bytes=%d-' % resume_len)
+ open_mode = 'ab'
+
+ # Establish connection
+ try:
+ data = urllib2.urlopen(request)
+ except (urllib2.HTTPError, ), err:
+ if err.code != 416: # 416 is 'Requested range not satisfiable'
+ raise
+ # Unable to resume
+ data = urllib2.urlopen(basic_request)
+ content_length = data.info()['Content-Length']
+
+ if content_length is not None and long(content_length) == resume_len:
+ # Because the file had already been fully downloaded
+ self.report_file_already_downloaded(filename)
+ return True
+ else:
+ # Because the server didn't let us
+ self.report_unable_to_resume()
+ open_mode = 'wb'
+
+ data_len = data.info().get('Content-length', None)
+ data_len_str = self.format_bytes(data_len)
+ byte_counter = 0
+ block_size = 1024
+ start = time.time()
+ while True:
+ # Download and write
+ before = time.time()
+ data_block = data.read(block_size)
+ after = time.time()
+ data_block_len = len(data_block)
+ if data_block_len == 0:
+ break
+ byte_counter += data_block_len
+
+ # Open file just in time
+ if stream is None:
+ try:
+ (stream, filename) = sanitize_open(filename, open_mode)
+ self.report_destination(filename)
+ except (OSError, IOError), err:
+ self.trouble('ERROR: unable to open for writing: %s' % str(err))
+ return False
+ stream.write(data_block)
+ block_size = self.best_block_size(after - before, data_block_len)
+
+ # Progress message
+ percent_str = self.calc_percent(byte_counter, data_len)
+ eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
+ speed_str = self.calc_speed(start, time.time(), byte_counter)
+ self.report_progress(percent_str, data_len_str, speed_str, eta_str)
+
+ # Apply rate limit
+ self.slow_down(start, byte_counter)
+
+ self.report_finish()
+ if data_len is not None and str(byte_counter) != data_len:
+ raise ContentTooShortError(byte_counter, long(data_len))
+ return True
+
+class InfoExtractor(object):
+ """Information Extractor class.
+
+ Information extractors are the classes that, given a URL, extract
+ information from the video (or videos) the URL refers to. This
+ information includes the real video URL, the video title and simplified
+ title, author and others. The information is stored in a dictionary
+ which is then passed to the FileDownloader. The FileDownloader
+ processes this information possibly downloading the video to the file
+ system, among other possible outcomes. The dictionaries must include
+ the following fields:
+
+ id: Video identifier.
+ url: Final video URL.
+ uploader: Nickname of the video uploader.
+ title: Literal title.
+ stitle: Simplified title.
+ ext: Video filename extension.
+
+ Subclasses of this one should re-define the _real_initialize() and
+ _real_extract() methods, as well as the suitable() static method.
+ Probably, they should also be instantiated and added to the main
+ downloader.
+ """
+
+ _ready = False
+ _downloader = None
+
+ def __init__(self, downloader=None):
+ """Constructor. Receives an optional downloader."""
+ self._ready = False
+ self.set_downloader(downloader)
+
+ @staticmethod
+ def suitable(url):
+ """Receives a URL and returns True if suitable for this IE."""
+ return False
+
+ def initialize(self):
+ """Initializes an instance (authentication, etc)."""
+ if not self._ready:
+ self._real_initialize()
+ self._ready = True
+
+ def extract(self, url):
+ """Extracts URL information and returns it in list of dicts."""
+ self.initialize()
+ return self._real_extract(url)
+
+ def set_downloader(self, downloader):
+ """Sets the downloader for this IE."""
+ self._downloader = downloader
+
+ def _real_initialize(self):
+ """Real initialization process. Redefine in subclasses."""
+ pass
+
+ def _real_extract(self, url):
+ """Real extraction process. Redefine in subclasses."""
+ pass
+
+class YoutubeIE(InfoExtractor):
+ """Information extractor for youtube.com."""
+
+ _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?[\?#](?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
+ _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
+ _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
+ _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
+ _NETRC_MACHINE = 'youtube'
+ _available_formats = ['37', '22', '35', '18', '5', '17', '13', None] # listed in order of priority for -b flag
+ _video_extensions = {
+ '13': '3gp',
+ '17': 'mp4',
+ '18': 'mp4',
+ '22': 'mp4',
+ '37': 'mp4',
+ }
+
+ @staticmethod
+ def suitable(url):
+ return (re.match(YoutubeIE._VALID_URL, url) is not None)
+
+ def report_lang(self):
+ """Report attempt to set language."""
+ self._downloader.to_stdout(u'[youtube] Setting language')
+
+ def report_login(self):
+ """Report attempt to log in."""
+ self._downloader.to_stdout(u'[youtube] Logging in')
+
+ def report_age_confirmation(self):
+ """Report attempt to confirm age."""
+ self._downloader.to_stdout(u'[youtube] Confirming age')
+
+ def report_video_info_webpage_download(self, video_id):
+ """Report attempt to download video info webpage."""
+ self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
+
+ def report_information_extraction(self, video_id):
+ """Report attempt to extract video information."""
+ self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
+
+ def report_unavailable_format(self, video_id, format):
+ """Report extracted video URL."""
+ self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
+
+ def report_rtmp_download(self):
+ """Indicate the download will use the RTMP protocol."""
+ self._downloader.to_stdout(u'[youtube] RTMP download detected')
+
+ def _real_initialize(self):
+ if self._downloader is None:
+ return
+
+ username = None
+ password = None
+ downloader_params = self._downloader.params
+
+ # Attempt to use provided username and password or .netrc data
+ if downloader_params.get('username', None) is not None:
+ username = downloader_params['username']
+ password = downloader_params['password']
+ elif downloader_params.get('usenetrc', False):
+ try:
+ info = netrc.netrc().authenticators(self._NETRC_MACHINE)
+ if info is not None:
+ username = info[0]
+ password = info[2]
+ else:
+ raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
+ except (IOError, netrc.NetrcParseError), err:
+ self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
+ return
+
+ # Set language
+ request = urllib2.Request(self._LANG_URL, None, std_headers)
+ try:
+ self.report_lang()
+ urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
+ return
+
+ # No authentication to be performed
+ if username is None:
+ return
+
Rapsys Git / youtubedl / blobdiff