]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube-dl
Imported Debian patch 2010.11.19-1
[youtubedl] / youtube-dl
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 # Author: Ricardo Garcia Gonzalez
4 # Author: Danny Colligan
5 # Author: Benjamin Johnson
6 # License: Public domain code
7 import cookielib
8 import htmlentitydefs
9 import httplib
10 import locale
11 import math
12 import netrc
13 import os
14 import os.path
15 import re
16 import socket
17 import string
18 import subprocess
19 import sys
20 import time
21 import urllib
22 import urllib2
23
24 # parse_qs was moved from the cgi module to the urlparse module recently.
25 try:
26 from urlparse import parse_qs
27 except ImportError:
28 from cgi import parse_qs
29
30 std_headers = {
31 'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12',
32 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
33 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
34 'Accept-Language': 'en-us,en;q=0.5',
35 }
36
37 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
38
39 month_name_to_number = {
40 'January': '01',
41 'February': '02',
42 'March': '03',
43 'April': '04',
44 'May': '05',
45 'June': '06',
46 'July': '07',
47 'August': '08',
48 'September': '09',
49 'October': '10',
50 'November': '11',
51 'December': '12',
52 }
53
54 def preferredencoding():
55 """Get preferred encoding.
56
57 Returns the best encoding scheme for the system, based on
58 locale.getpreferredencoding() and some further tweaks.
59 """
60 def yield_preferredencoding():
61 try:
62 pref = locale.getpreferredencoding()
63 u'TEST'.encode(pref)
64 except:
65 pref = 'UTF-8'
66 while True:
67 yield pref
68 return yield_preferredencoding().next()
69
70 def htmlentity_transform(matchobj):
71 """Transforms an HTML entity to a Unicode character.
72
73 This function receives a match object and is intended to be used with
74 the re.sub() function.
75 """
76 entity = matchobj.group(1)
77
78 # Known non-numeric HTML entity
79 if entity in htmlentitydefs.name2codepoint:
80 return unichr(htmlentitydefs.name2codepoint[entity])
81
82 # Unicode character
83 mobj = re.match(ur'(?u)#(x?\d+)', entity)
84 if mobj is not None:
85 numstr = mobj.group(1)
86 if numstr.startswith(u'x'):
87 base = 16
88 numstr = u'0%s' % numstr
89 else:
90 base = 10
91 return unichr(long(numstr, base))
92
93 # Unknown entity in name, return its literal representation
94 return (u'&%s;' % entity)
95
96 def sanitize_title(utitle):
97 """Sanitizes a video title so it could be used as part of a filename."""
98 utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
99 return utitle.replace(unicode(os.sep), u'%')
100
101 def sanitize_open(filename, open_mode):
102 """Try to open the given filename, and slightly tweak it if this fails.
103
104 Attempts to open the given filename. If this fails, it tries to change
105 the filename slightly, step by step, until it's either able to open it
106 or it fails and raises a final exception, like the standard open()
107 function.
108
109 It returns the tuple (stream, definitive_file_name).
110 """
111 try:
112 if filename == u'-':
113 if sys.platform == 'win32':
114 import msvcrt
115 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
116 return (sys.stdout, filename)
117 stream = open(filename, open_mode)
118 return (stream, filename)
119 except (IOError, OSError), err:
120 # In case of error, try to remove win32 forbidden chars
121 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
122
123 # An exception here should be caught in the caller
124 stream = open(filename, open_mode)
125 return (stream, filename)
126
127
128 class DownloadError(Exception):
129 """Download Error exception.
130
131 This exception may be thrown by FileDownloader objects if they are not
132 configured to continue on errors. They will contain the appropriate
133 error message.
134 """
135 pass
136
137 class SameFileError(Exception):
138 """Same File exception.
139
140 This exception will be thrown by FileDownloader objects if they detect
141 multiple files would have to be downloaded to the same file on disk.
142 """
143 pass
144
145 class PostProcessingError(Exception):
146 """Post Processing exception.
147
148 This exception may be raised by PostProcessor's .run() method to
149 indicate an error in the postprocessing task.
150 """
151 pass
152
153 class UnavailableVideoError(Exception):
154 """Unavailable Format exception.
155
156 This exception will be thrown when a video is requested
157 in a format that is not available for that video.
158 """
159 pass
160
161 class ContentTooShortError(Exception):
162 """Content Too Short exception.
163
164 This exception may be raised by FileDownloader objects when a file they
165 download is too small for what the server announced first, indicating
166 the connection was probably interrupted.
167 """
168 # Both in bytes
169 downloaded = None
170 expected = None
171
172 def __init__(self, downloaded, expected):
173 self.downloaded = downloaded
174 self.expected = expected
175
176 class FileDownloader(object):
177 """File Downloader class.
178
179 File downloader objects are the ones responsible of downloading the
180 actual video file and writing it to disk if the user has requested
181 it, among some other tasks. In most cases there should be one per
182 program. As, given a video URL, the downloader doesn't know how to
183 extract all the needed information, task that InfoExtractors do, it
184 has to pass the URL to one of them.
185
186 For this, file downloader objects have a method that allows
187 InfoExtractors to be registered in a given order. When it is passed
188 a URL, the file downloader handles it to the first InfoExtractor it
189 finds that reports being able to handle it. The InfoExtractor extracts
190 all the information about the video or videos the URL refers to, and
191 asks the FileDownloader to process the video information, possibly
192 downloading the video.
193
194 File downloaders accept a lot of parameters. In order not to saturate
195 the object constructor with arguments, it receives a dictionary of
196 options instead. These options are available through the params
197 attribute for the InfoExtractors to use. The FileDownloader also
198 registers itself as the downloader in charge for the InfoExtractors
199 that are added to it, so this is a "mutual registration".
200
201 Available options:
202
203 username: Username for authentication purposes.
204 password: Password for authentication purposes.
205 usenetrc: Use netrc for authentication instead.
206 quiet: Do not print messages to stdout.
207 forceurl: Force printing final URL.
208 forcetitle: Force printing title.
209 forcethumbnail: Force printing thumbnail URL.
210 forcedescription: Force printing description.
211 simulate: Do not download the video files.
212 format: Video format code.
213 format_limit: Highest quality format to try.
214 outtmpl: Template for output names.
215 ignoreerrors: Do not stop on download errors.
216 ratelimit: Download speed limit, in bytes/sec.
217 nooverwrites: Prevent overwriting files.
218 retries: Number of times to retry for HTTP error 5xx
219 continuedl: Try to continue downloads if possible.
220 noprogress: Do not print the progress bar.
221 playliststart: Playlist item to start at.
222 playlistend: Playlist item to end at.
223 logtostderr: Log messages to stderr instead of stdout.
224 """
225
226 params = None
227 _ies = []
228 _pps = []
229 _download_retcode = None
230 _num_downloads = None
231 _screen_file = None
232
233 def __init__(self, params):
234 """Create a FileDownloader object with the given options."""
235 self._ies = []
236 self._pps = []
237 self._download_retcode = 0
238 self._num_downloads = 0
239 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
240 self.params = params
241
242 @staticmethod
243 def pmkdir(filename):
244 """Create directory components in filename. Similar to Unix "mkdir -p"."""
245 components = filename.split(os.sep)
246 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
247 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
248 for dir in aggregate:
249 if not os.path.exists(dir):
250 os.mkdir(dir)
251
252 @staticmethod
253 def format_bytes(bytes):
254 if bytes is None:
255 return 'N/A'
256 if type(bytes) is str:
257 bytes = float(bytes)
258 if bytes == 0.0:
259 exponent = 0
260 else:
261 exponent = long(math.log(bytes, 1024.0))
262 suffix = 'bkMGTPEZY'[exponent]
263 converted = float(bytes) / float(1024**exponent)
264 return '%.2f%s' % (converted, suffix)
265
266 @staticmethod
267 def calc_percent(byte_counter, data_len):
268 if data_len is None:
269 return '---.-%'
270 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
271
272 @staticmethod
273 def calc_eta(start, now, total, current):
274 if total is None:
275 return '--:--'
276 dif = now - start
277 if current == 0 or dif < 0.001: # One millisecond
278 return '--:--'
279 rate = float(current) / dif
280 eta = long((float(total) - float(current)) / rate)
281 (eta_mins, eta_secs) = divmod(eta, 60)
282 if eta_mins > 99:
283 return '--:--'
284 return '%02d:%02d' % (eta_mins, eta_secs)
285
286 @staticmethod
287 def calc_speed(start, now, bytes):
288 dif = now - start
289 if bytes == 0 or dif < 0.001: # One millisecond
290 return '%10s' % '---b/s'
291 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
292
293 @staticmethod
294 def best_block_size(elapsed_time, bytes):
295 new_min = max(bytes / 2.0, 1.0)
296 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
297 if elapsed_time < 0.001:
298 return long(new_max)
299 rate = bytes / elapsed_time
300 if rate > new_max:
301 return long(new_max)
302 if rate < new_min:
303 return long(new_min)
304 return long(rate)
305
306 @staticmethod
307 def parse_bytes(bytestr):
308 """Parse a string indicating a byte quantity into a long integer."""
309 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
310 if matchobj is None:
311 return None
312 number = float(matchobj.group(1))
313 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
314 return long(round(number * multiplier))
315
316 def add_info_extractor(self, ie):
317 """Add an InfoExtractor object to the end of the list."""
318 self._ies.append(ie)
319 ie.set_downloader(self)
320
321 def add_post_processor(self, pp):
322 """Add a PostProcessor object to the end of the chain."""
323 self._pps.append(pp)
324 pp.set_downloader(self)
325
326 def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
327 """Print message to stdout if not in quiet mode."""
328 try:
329 if not self.params.get('quiet', False):
330 terminator = [u'\n', u''][skip_eol]
331 print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()),
332 self._screen_file.flush()
333 except (UnicodeEncodeError), err:
334 if not ignore_encoding_errors:
335 raise
336
337 def to_stderr(self, message):
338 """Print message to stderr."""
339 print >>sys.stderr, message.encode(preferredencoding())
340
341 def fixed_template(self):
342 """Checks if the output template is fixed."""
343 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
344
345 def trouble(self, message=None):
346 """Determine action to take when a download problem appears.
347
348 Depending on if the downloader has been configured to ignore
349 download errors or not, this method may throw an exception or
350 not when errors are found, after printing the message.
351 """
352 if message is not None:
353 self.to_stderr(message)
354 if not self.params.get('ignoreerrors', False):
355 raise DownloadError(message)
356 self._download_retcode = 1
357
358 def slow_down(self, start_time, byte_counter):
359 """Sleep if the download speed is over the rate limit."""
360 rate_limit = self.params.get('ratelimit', None)
361 if rate_limit is None or byte_counter == 0:
362 return
363 now = time.time()
364 elapsed = now - start_time
365 if elapsed <= 0.0:
366 return
367 speed = float(byte_counter) / elapsed
368 if speed > rate_limit:
369 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
370
371 def report_destination(self, filename):
372 """Report destination filename."""
373 self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
374
375 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
376 """Report download progress."""
377 if self.params.get('noprogress', False):
378 return
379 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
380 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
381
382 def report_resuming_byte(self, resume_len):
383 """Report attempt to resume at given byte."""
384 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
385
386 def report_retry(self, count, retries):
387 """Report retry in case of HTTP error 5xx"""
388 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
389
390 def report_file_already_downloaded(self, file_name):
391 """Report file has already been fully downloaded."""
392 try:
393 self.to_screen(u'[download] %s has already been downloaded' % file_name)
394 except (UnicodeEncodeError), err:
395 self.to_screen(u'[download] The file has already been downloaded')
396
397 def report_unable_to_resume(self):
398 """Report it was impossible to resume download."""
399 self.to_screen(u'[download] Unable to resume')
400
401 def report_finish(self):
402 """Report download finished."""
403 if self.params.get('noprogress', False):
404 self.to_screen(u'[download] Download completed')
405 else:
406 self.to_screen(u'')
407
408 def increment_downloads(self):
409 """Increment the ordinal that assigns a number to each file."""
410 self._num_downloads += 1
411
412 def process_info(self, info_dict):
413 """Process a single dictionary returned by an InfoExtractor."""
414 # Do nothing else if in simulate mode
415 if self.params.get('simulate', False):
416 # Forced printings
417 if self.params.get('forcetitle', False):
418 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
419 if self.params.get('forceurl', False):
420 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
421 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
422 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
423 if self.params.get('forcedescription', False) and 'description' in info_dict:
424 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
425
426 return
427
428 try:
429 template_dict = dict(info_dict)
430 template_dict['epoch'] = unicode(long(time.time()))
431 template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
432 filename = self.params['outtmpl'] % template_dict
433 except (ValueError, KeyError), err:
434 self.trouble(u'ERROR: invalid system charset or erroneous output template')
435 return
436 if self.params.get('nooverwrites', False) and os.path.exists(filename):
437 self.to_stderr(u'WARNING: file exists and will be skipped')
438 return
439
440 try:
441 self.pmkdir(filename)
442 except (OSError, IOError), err:
443 self.trouble(u'ERROR: unable to create directories: %s' % str(err))
444 return
445
446 try:
447 success = self._do_download(filename, info_dict['url'].encode('utf-8'), info_dict.get('player_url', None))
448 except (OSError, IOError), err:
449 raise UnavailableVideoError
450 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
451 self.trouble(u'ERROR: unable to download video data: %s' % str(err))
452 return
453 except (ContentTooShortError, ), err:
454 self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
455 return
456
457 if success:
458 try:
459 self.post_process(filename, info_dict)
460 except (PostProcessingError), err:
461 self.trouble(u'ERROR: postprocessing: %s' % str(err))
462 return
463
464 def download(self, url_list):
465 """Download a given list of URLs."""
466 if len(url_list) > 1 and self.fixed_template():
467 raise SameFileError(self.params['outtmpl'])
468
469 for url in url_list:
470 suitable_found = False
471 for ie in self._ies:
472 # Go to next InfoExtractor if not suitable
473 if not ie.suitable(url):
474 continue
475
476 # Suitable InfoExtractor found
477 suitable_found = True
478
479 # Extract information from URL and process it
480 ie.extract(url)
481
482 # Suitable InfoExtractor had been found; go to next URL
483 break
484
485 if not suitable_found:
486 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
487
488 return self._download_retcode
489
490 def post_process(self, filename, ie_info):
491 """Run the postprocessing chain on the given file."""
492 info = dict(ie_info)
493 info['filepath'] = filename
494 for pp in self._pps:
495 info = pp.run(info)
496 if info is None:
497 break
498
499 def _download_with_rtmpdump(self, filename, url, player_url):
500 self.report_destination(filename)
501
502 # Check for rtmpdump first
503 try:
504 subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
505 except (OSError, IOError):
506 self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
507 return False
508
509 # Download using rtmpdump. rtmpdump returns exit code 2 when
510 # the connection was interrumpted and resuming appears to be
511 # possible. This is part of rtmpdump's normal usage, AFAIK.
512 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', filename]
513 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
514 while retval == 2 or retval == 1:
515 prevsize = os.path.getsize(filename)
516 self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
517 time.sleep(5.0) # This seems to be needed
518 retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
519 cursize = os.path.getsize(filename)
520 if prevsize == cursize and retval == 1:
521 break
522 if retval == 0:
523 self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
524 return True
525 else:
526 self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
527 return False
528
529 def _do_download(self, filename, url, player_url):
530 # Attempt to download using rtmpdump
531 if url.startswith('rtmp'):
532 return self._download_with_rtmpdump(filename, url, player_url)
533
534 stream = None
535 open_mode = 'wb'
536 basic_request = urllib2.Request(url, None, std_headers)
537 request = urllib2.Request(url, None, std_headers)
538
539 # Establish possible resume length
540 if os.path.isfile(filename):
541 resume_len = os.path.getsize(filename)
542 else:
543 resume_len = 0
544
545 # Request parameters in case of being able to resume
546 if self.params.get('continuedl', False) and resume_len != 0:
547 self.report_resuming_byte(resume_len)
548 request.add_header('Range','bytes=%d-' % resume_len)
549 open_mode = 'ab'
550
551 count = 0
552 retries = self.params.get('retries', 0)
553 while count <= retries:
554 # Establish connection
555 try:
556 data = urllib2.urlopen(request)
557 break
558 except (urllib2.HTTPError, ), err:
559 if (err.code < 500 or err.code >= 600) and err.code != 416:
560 # Unexpected HTTP error
561 raise
562 elif err.code == 416:
563 # Unable to resume (requested range not satisfiable)
564 try:
565 # Open the connection again without the range header
566 data = urllib2.urlopen(basic_request)
567 content_length = data.info()['Content-Length']
568 except (urllib2.HTTPError, ), err:
569 if err.code < 500 or err.code >= 600:
570 raise
571 else:
572 # Examine the reported length
573 if (content_length is not None and
574 (resume_len - 100 < long(content_length) < resume_len + 100)):
575 # The file had already been fully downloaded.
576 # Explanation to the above condition: in issue #175 it was revealed that
577 # YouTube sometimes adds or removes a few bytes from the end of the file,
578 # changing the file size slightly and causing problems for some users. So
579 # I decided to implement a suggested change and consider the file
580 # completely downloaded if the file size differs less than 100 bytes from
581 # the one in the hard drive.
582 self.report_file_already_downloaded(filename)
583 return True
584 else:
585 # The length does not match, we start the download over
586 self.report_unable_to_resume()
587 open_mode = 'wb'
588 break
589 # Retry
590 count += 1
591 if count <= retries:
592 self.report_retry(count, retries)
593
594 if count > retries:
595 self.trouble(u'ERROR: giving up after %s retries' % retries)
596 return False
597
598 data_len = data.info().get('Content-length', None)
599 data_len_str = self.format_bytes(data_len)
600 byte_counter = 0
601 block_size = 1024
602 start = time.time()
603 while True:
604 # Download and write
605 before = time.time()
606 data_block = data.read(block_size)
607 after = time.time()
608 data_block_len = len(data_block)
609 if data_block_len == 0:
610 break
611 byte_counter += data_block_len
612
613 # Open file just in time
614 if stream is None:
615 try:
616 (stream, filename) = sanitize_open(filename, open_mode)
617 self.report_destination(filename)
618 except (OSError, IOError), err:
619 self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
620 return False
621 try:
622 stream.write(data_block)
623 except (IOError, OSError), err:
624 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
625 return False
626 block_size = self.best_block_size(after - before, data_block_len)
627
628 # Progress message
629 percent_str = self.calc_percent(byte_counter, data_len)
630 eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
631 speed_str = self.calc_speed(start, time.time(), byte_counter)
632 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
633
634 # Apply rate limit
635 self.slow_down(start, byte_counter)
636
637 self.report_finish()
638 if data_len is not None and str(byte_counter) != data_len:
639 raise ContentTooShortError(byte_counter, long(data_len))
640 return True
641
642 class InfoExtractor(object):
643 """Information Extractor class.
644
645 Information extractors are the classes that, given a URL, extract
646 information from the video (or videos) the URL refers to. This
647 information includes the real video URL, the video title and simplified
648 title, author and others. The information is stored in a dictionary
649 which is then passed to the FileDownloader. The FileDownloader
650 processes this information possibly downloading the video to the file
651 system, among other possible outcomes. The dictionaries must include
652 the following fields:
653
654 id: Video identifier.
655 url: Final video URL.
656 uploader: Nickname of the video uploader.
657 title: Literal title.
658 stitle: Simplified title.
659 ext: Video filename extension.
660 format: Video format.
661 player_url: SWF Player URL (may be None).
662
663 The following fields are optional. Their primary purpose is to allow
664 youtube-dl to serve as the backend for a video search function, such
665 as the one in youtube2mp3. They are only used when their respective
666 forced printing functions are called:
667
668 thumbnail: Full URL to a video thumbnail image.
669 description: One-line video description.
670
671 Subclasses of this one should re-define the _real_initialize() and
672 _real_extract() methods, as well as the suitable() static method.
673 Probably, they should also be instantiated and added to the main
674 downloader.
675 """
676
677 _ready = False
678 _downloader = None
679
680 def __init__(self, downloader=None):
681 """Constructor. Receives an optional downloader."""
682 self._ready = False
683 self.set_downloader(downloader)
684
685 @staticmethod
686 def suitable(url):
687 """Receives a URL and returns True if suitable for this IE."""
688 return False
689
690 def initialize(self):
691 """Initializes an instance (authentication, etc)."""
692 if not self._ready:
693 self._real_initialize()
694 self._ready = True
695
696 def extract(self, url):
697 """Extracts URL information and returns it in list of dicts."""
698 self.initialize()
699 return self._real_extract(url)
700
701 def set_downloader(self, downloader):
702 """Sets the downloader for this IE."""
703 self._downloader = downloader
704
705 def _real_initialize(self):
706 """Real initialization process. Redefine in subclasses."""
707 pass
708
709 def _real_extract(self, url):
710 """Real extraction process. Redefine in subclasses."""
711 pass
712
713 class YoutubeIE(InfoExtractor):
714 """Information extractor for youtube.com."""
715
716 _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$'
717 _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
718 _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
719 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
720 _NETRC_MACHINE = 'youtube'
721 # Listed in order of quality
722 _available_formats = ['38', '37', '22', '45', '35', '34', '43', '18', '6', '5', '17', '13']
723 _video_extensions = {
724 '13': '3gp',
725 '17': 'mp4',
726 '18': 'mp4',
727 '22': 'mp4',
728 '37': 'mp4',
729 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
730 '43': 'webm',
731 '45': 'webm',
732 }
733
734 @staticmethod
735 def suitable(url):
736 return (re.match(YoutubeIE._VALID_URL, url) is not None)
737
738 def report_lang(self):
739 """Report attempt to set language."""
740 self._downloader.to_screen(u'[youtube] Setting language')
741
742 def report_login(self):
743 """Report attempt to log in."""
744 self._downloader.to_screen(u'[youtube] Logging in')
745
746 def report_age_confirmation(self):
747 """Report attempt to confirm age."""
748 self._downloader.to_screen(u'[youtube] Confirming age')
749
750 def report_video_webpage_download(self, video_id):
751 """Report attempt to download video webpage."""
752 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
753
754 def report_video_info_webpage_download(self, video_id):
755 """Report attempt to download video info webpage."""
756 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
757
758 def report_information_extraction(self, video_id):
759 """Report attempt to extract video information."""
760 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
761
762 def report_unavailable_format(self, video_id, format):
763 """Report extracted video URL."""
764 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
765
766 def report_rtmp_download(self):
767 """Indicate the download will use the RTMP protocol."""
768 self._downloader.to_screen(u'[youtube] RTMP download detected')
769
770 def _real_initialize(self):
771 if self._downloader is None:
772 return
773
774 username = None
775 password = None
776 downloader_params = self._downloader.params
777
778 # Attempt to use provided username and password or .netrc data
779 if downloader_params.get('username', None) is not None:
780 username = downloader_params['username']
781 password = downloader_params['password']
782 elif downloader_params.get('usenetrc', False):
783 try:
784 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
785 if info is not None:
786 username = info[0]
787 password = info[2]
788 else:
789 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
790 except (IOError, netrc.NetrcParseError), err:
791 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
792 return
793
794 # Set language
795 request = urllib2.Request(self._LANG_URL, None, std_headers)
796 try:
797 self.report_lang()
798 urllib2.urlopen(request).read()
799 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
800 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
801 return
802
803 # No authentication to be performed
804 if username is None:
805 return
806
807 # Log in
808 login_form = {
809 'current_form': 'loginForm',
810 'next': '/',
811 'action_login': 'Log In',
812 'username': username,
813 'password': password,
814 }
815 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
816 try:
817 self.report_login()
818 login_results = urllib2.urlopen(request).read()
819 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
820 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
821 return
822 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
823 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
824 return
825
826 # Confirm age
827 age_form = {
828 'next_url': '/',
829 'action_confirm': 'Confirm',
830 }
831 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
832 try:
833 self.report_age_confirmation()
834 age_results = urllib2.urlopen(request).read()
835 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
836 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
837 return
838
839 def _real_extract(self, url):
840 # Extract video id from URL
841 mobj = re.match(self._VALID_URL, url)
842 if mobj is None:
843 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
844 return
845 video_id = mobj.group(2)
846
847 # Get video webpage
848 self.report_video_webpage_download(video_id)
849 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&amp;has_verified=1' % video_id, None, std_headers)
850 try:
851 video_webpage = urllib2.urlopen(request).read()
852 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
853 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
854 return
855
856 # Attempt to extract SWF player URL
857 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
858 if mobj is not None:
859 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
860 else:
861 player_url = None
862
863 # Get video info
864 self.report_video_info_webpage_download(video_id)
865 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
866 video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
867 % (video_id, el_type))
868 request = urllib2.Request(video_info_url, None, std_headers)
869 try:
870 video_info_webpage = urllib2.urlopen(request).read()
871 video_info = parse_qs(video_info_webpage)
872 if 'token' in video_info:
873 break
874 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
875 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
876 return
877 if 'token' not in video_info:
878 if 'reason' in video_info:
879 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
880 else:
881 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
882 return
883
884 # Start extracting information
885 self.report_information_extraction(video_id)
886
887 # uploader
888 if 'author' not in video_info:
889 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
890 return
891 video_uploader = urllib.unquote_plus(video_info['author'][0])
892
893 # title
894 if 'title' not in video_info:
895 self._downloader.trouble(u'ERROR: unable to extract video title')
896 return
897 video_title = urllib.unquote_plus(video_info['title'][0])
898 video_title = video_title.decode('utf-8')
899 video_title = sanitize_title(video_title)
900
901 # simplified title
902 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
903 simple_title = simple_title.strip(ur'_')
904
905 # thumbnail image
906 if 'thumbnail_url' not in video_info:
907 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
908 video_thumbnail = ''
909 else: # don't panic if we can't find it
910 video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
911
912 # upload date
913 upload_date = u'NA'
914 mobj = re.search(r'id="eow-date".*?>(.*?)</span>', video_webpage, re.DOTALL)
915 if mobj is not None:
916 try:
917 if ',' in mobj.group(1):
918 # Month Day, Year
919 m, d, y = mobj.group(1).replace(',', '').split()
920 else:
921 # Day Month Year, we'll suppose
922 d, m, y = mobj.group(1).split()
923 m = month_name_to_number[m]
924 d = '%02d' % (long(d))
925 upload_date = '%s%s%s' % (y, m, d)
926 except:
927 upload_date = u'NA'
928
929 # description
930 video_description = 'No description available.'
931 if self._downloader.params.get('forcedescription', False):
932 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
933 if mobj is not None:
934 video_description = mobj.group(1)
935
936 # token
937 video_token = urllib.unquote_plus(video_info['token'][0])
938
939 # Decide which formats to download
940 requested_format = self._downloader.params.get('format', None)
941 get_video_template = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=&ps=&asv=&fmt=%%s' % (video_id, video_token)
942
943 if 'fmt_url_map' in video_info:
944 url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(','))
945 format_limit = self._downloader.params.get('format_limit', None)
946 if format_limit is not None and format_limit in self._available_formats:
947 format_list = self._available_formats[self._available_formats.index(format_limit):]
948 else:
949 format_list = self._available_formats
950 existing_formats = [x for x in format_list if x in url_map]
951 if len(existing_formats) == 0:
952 self._downloader.trouble(u'ERROR: no known formats available for video')
953 return
954 if requested_format is None:
955 video_url_list = [(existing_formats[0], get_video_template % existing_formats[0])] # Best quality
956 elif requested_format == '-1':
957 video_url_list = [(f, get_video_template % f) for f in existing_formats] # All formats
958 else:
959 video_url_list = [(requested_format, get_video_template % requested_format)] # Specific format
960
961 elif 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
962 self.report_rtmp_download()
963 video_url_list = [(None, video_info['conn'][0])]
964
965 else:
966 self._downloader.trouble(u'ERROR: no fmt_url_map or conn information found in video info')
967 return
968
969 for format_param, video_real_url in video_url_list:
970 # At this point we have a new video
971 self._downloader.increment_downloads()
972
973 # Extension
974 video_extension = self._video_extensions.get(format_param, 'flv')
975
976 # Find the video URL in fmt_url_map or conn paramters
977 try:
978 # Process video information
979 self._downloader.process_info({
980 'id': video_id.decode('utf-8'),
981 'url': video_real_url.decode('utf-8'),
982 'uploader': video_uploader.decode('utf-8'),
983 'upload_date': upload_date,
984 'title': video_title,
985 'stitle': simple_title,
986 'ext': video_extension.decode('utf-8'),
987 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
988 'thumbnail': video_thumbnail.decode('utf-8'),
989 'description': video_description.decode('utf-8'),
990 'player_url': player_url,
991 })
992 except UnavailableVideoError, err:
993 self._downloader.trouble(u'ERROR: unable to download video (format may not be available)')
994
995
996 class MetacafeIE(InfoExtractor):
997 """Information Extractor for metacafe.com."""
998
999 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
1000 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
1001 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
1002 _youtube_ie = None
1003
1004 def __init__(self, youtube_ie, downloader=None):
1005 InfoExtractor.__init__(self, downloader)
1006 self._youtube_ie = youtube_ie
1007
1008 @staticmethod
1009 def suitable(url):
1010 return (re.match(MetacafeIE._VALID_URL, url) is not None)
1011
1012 def report_disclaimer(self):
1013 """Report disclaimer retrieval."""
1014 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
1015
1016 def report_age_confirmation(self):
1017 """Report attempt to confirm age."""
1018 self._downloader.to_screen(u'[metacafe] Confirming age')
1019
1020 def report_download_webpage(self, video_id):
1021 """Report webpage download."""
1022 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
1023
1024 def report_extraction(self, video_id):
1025 """Report information extraction."""
1026 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
1027
1028 def _real_initialize(self):
1029 # Retrieve disclaimer
1030 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
1031 try:
1032 self.report_disclaimer()
1033 disclaimer = urllib2.urlopen(request).read()
1034 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1035 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1036 return
1037
1038 # Confirm age
1039 disclaimer_form = {
1040 'filters': '0',
1041 'submit': "Continue - I'm over 18",
1042 }
1043 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
1044 try:
1045 self.report_age_confirmation()
1046 disclaimer = urllib2.urlopen(request).read()
1047 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1048 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1049 return
1050
1051 def _real_extract(self, url):
1052 # Extract id and simplified title from URL
1053 mobj = re.match(self._VALID_URL, url)
1054 if mobj is None:
1055 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1056 return
1057
1058 video_id = mobj.group(1)
1059
1060 # Check if video comes from YouTube
1061 mobj2 = re.match(r'^yt-(.*)$', video_id)
1062 if mobj2 is not None:
1063 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1064 return
1065
1066 # At this point we have a new video
1067 self._downloader.increment_downloads()
1068
1069 simple_title = mobj.group(2).decode('utf-8')
1070
1071 # Retrieve video webpage to extract further information
1072 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1073 try:
1074 self.report_download_webpage(video_id)
1075 webpage = urllib2.urlopen(request).read()
1076 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1077 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1078 return
1079
1080 # Extract URL, uploader and title from webpage
1081 self.report_extraction(video_id)
1082 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1083 if mobj is not None:
1084 mediaURL = urllib.unquote(mobj.group(1))
1085 video_extension = mediaURL[-3:]
1086
1087 # Extract gdaKey if available
1088 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1089 if mobj is None:
1090 video_url = mediaURL
1091 else:
1092 gdaKey = mobj.group(1)
1093 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1094 else:
1095 mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
1096 if mobj is None:
1097 self._downloader.trouble(u'ERROR: unable to extract media URL')
1098 return
1099 vardict = parse_qs(mobj.group(1))
1100 if 'mediaData' not in vardict:
1101 self._downloader.trouble(u'ERROR: unable to extract media URL')
1102 return
1103 mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
1104 if mobj is None:
1105 self._downloader.trouble(u'ERROR: unable to extract media URL')
1106 return
1107 mediaURL = mobj.group(1).replace('\\/', '/')
1108 video_extension = mediaURL[-3:]
1109 video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
1110
1111 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1112 if mobj is None:
1113 self._downloader.trouble(u'ERROR: unable to extract title')
1114 return
1115 video_title = mobj.group(1).decode('utf-8')
1116 video_title = sanitize_title(video_title)
1117
1118 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1119 if mobj is None:
1120 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1121 return
1122 video_uploader = mobj.group(1)
1123
1124 try:
1125 # Process video information
1126 self._downloader.process_info({
1127 'id': video_id.decode('utf-8'),
1128 'url': video_url.decode('utf-8'),
1129 'uploader': video_uploader.decode('utf-8'),
1130 'upload_date': u'NA',
1131 'title': video_title,
1132 'stitle': simple_title,
1133 'ext': video_extension.decode('utf-8'),
1134 'format': u'NA',
1135 'player_url': None,
1136 })
1137 except UnavailableVideoError:
1138 self._downloader.trouble(u'ERROR: unable to download video')
1139
1140
1141 class DailymotionIE(InfoExtractor):
1142 """Information Extractor for Dailymotion"""
1143
1144 _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1145
1146 def __init__(self, downloader=None):
1147 InfoExtractor.__init__(self, downloader)
1148
1149 @staticmethod
1150 def suitable(url):
1151 return (re.match(DailymotionIE._VALID_URL, url) is not None)
1152
1153 def report_download_webpage(self, video_id):
1154 """Report webpage download."""
1155 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
1156
1157 def report_extraction(self, video_id):
1158 """Report information extraction."""
1159 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
1160
1161 def _real_initialize(self):
1162 return
1163
1164 def _real_extract(self, url):
1165 # Extract id and simplified title from URL
1166 mobj = re.match(self._VALID_URL, url)
1167 if mobj is None:
1168 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1169 return
1170
1171 # At this point we have a new video
1172 self._downloader.increment_downloads()
1173 video_id = mobj.group(1)
1174
1175 simple_title = mobj.group(2).decode('utf-8')
1176 video_extension = 'flv'
1177
1178 # Retrieve video webpage to extract further information
1179 request = urllib2.Request(url)
1180 try:
1181 self.report_download_webpage(video_id)
1182 webpage = urllib2.urlopen(request).read()
1183 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1184 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1185 return
1186
1187 # Extract URL, uploader and title from webpage
1188 self.report_extraction(video_id)
1189 mobj = re.search(r'(?i)addVariable\(\"video\"\s*,\s*\"([^\"]*)\"\)', webpage)
1190 if mobj is None:
1191 self._downloader.trouble(u'ERROR: unable to extract media URL')
1192 return
1193 mediaURL = urllib.unquote(mobj.group(1))
1194
1195 # if needed add http://www.dailymotion.com/ if relative URL
1196
1197 video_url = mediaURL
1198
1199 # '<meta\s+name="title"\s+content="Dailymotion\s*[:\-]\s*(.*?)"\s*\/\s*>'
1200 mobj = re.search(r'(?im)<title>Dailymotion\s*[\-:]\s*(.+?)</title>', webpage)
1201 if mobj is None:
1202 self._downloader.trouble(u'ERROR: unable to extract title')
1203 return
1204 video_title = mobj.group(1).decode('utf-8')
1205 video_title = sanitize_title(video_title)
1206
1207 mobj = re.search(r'(?im)<div class="dmco_html owner">.*?<a class="name" href="/.+?">(.+?)</a>', webpage)
1208 if mobj is None:
1209 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1210 return
1211 video_uploader = mobj.group(1)
1212
1213 try:
1214 # Process video information
1215 self._downloader.process_info({
1216 'id': video_id.decode('utf-8'),
1217 'url': video_url.decode('utf-8'),
1218 'uploader': video_uploader.decode('utf-8'),
1219 'upload_date': u'NA',
1220 'title': video_title,
1221 'stitle': simple_title,
1222 'ext': video_extension.decode('utf-8'),
1223 'format': u'NA',
1224 'player_url': None,
1225 })
1226 except UnavailableVideoError:
1227 self._downloader.trouble(u'ERROR: unable to download video')
1228
1229 class GoogleIE(InfoExtractor):
1230 """Information extractor for video.google.com."""
1231
1232 _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1233
1234 def __init__(self, downloader=None):
1235 InfoExtractor.__init__(self, downloader)
1236
1237 @staticmethod
1238 def suitable(url):
1239 return (re.match(GoogleIE._VALID_URL, url) is not None)
1240
1241 def report_download_webpage(self, video_id):
1242 """Report webpage download."""
1243 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
1244
1245 def report_extraction(self, video_id):
1246 """Report information extraction."""
1247 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
1248
1249 def _real_initialize(self):
1250 return
1251
1252 def _real_extract(self, url):
1253 # Extract id from URL
1254 mobj = re.match(self._VALID_URL, url)
1255 if mobj is None:
1256 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1257 return
1258
1259 # At this point we have a new video
1260 self._downloader.increment_downloads()
1261 video_id = mobj.group(1)
1262
1263 video_extension = 'mp4'
1264
1265 # Retrieve video webpage to extract further information
1266 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1267 try:
1268 self.report_download_webpage(video_id)
1269 webpage = urllib2.urlopen(request).read()
1270 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1271 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1272 return
1273
1274 # Extract URL, uploader, and title from webpage
1275 self.report_extraction(video_id)
1276 mobj = re.search(r"download_url:'([^']+)'", webpage)
1277 if mobj is None:
1278 video_extension = 'flv'
1279 mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1280 if mobj is None:
1281 self._downloader.trouble(u'ERROR: unable to extract media URL')
1282 return
1283 mediaURL = urllib.unquote(mobj.group(1))
1284 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1285 mediaURL = mediaURL.replace('\\x26', '\x26')
1286
1287 video_url = mediaURL
1288
1289 mobj = re.search(r'<title>(.*)</title>', webpage)
1290 if mobj is None:
1291 self._downloader.trouble(u'ERROR: unable to extract title')
1292 return
1293 video_title = mobj.group(1).decode('utf-8')
1294 video_title = sanitize_title(video_title)
1295 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1296
1297 # Extract video description
1298 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1299 if mobj is None:
1300 self._downloader.trouble(u'ERROR: unable to extract video description')
1301 return
1302 video_description = mobj.group(1).decode('utf-8')
1303 if not video_description:
1304 video_description = 'No description available.'
1305
1306 # Extract video thumbnail
1307 if self._downloader.params.get('forcethumbnail', False):
1308 request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1309 try:
1310 webpage = urllib2.urlopen(request).read()
1311 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1312 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1313 return
1314 mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1315 if mobj is None:
1316 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1317 return
1318 video_thumbnail = mobj.group(1)
1319 else: # we need something to pass to process_info
1320 video_thumbnail = ''
1321
1322
1323 try:
1324 # Process video information
1325 self._downloader.process_info({
1326 'id': video_id.decode('utf-8'),
1327 'url': video_url.decode('utf-8'),
1328 'uploader': u'NA',
1329 'upload_date': u'NA',
1330 'title': video_title,
1331 'stitle': simple_title,
1332 'ext': video_extension.decode('utf-8'),
1333 'format': u'NA',
1334 'player_url': None,
1335 })
1336 except UnavailableVideoError:
1337 self._downloader.trouble(u'ERROR: unable to download video')
1338
1339
1340 class PhotobucketIE(InfoExtractor):
1341 """Information extractor for photobucket.com."""
1342
1343 _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1344
1345 def __init__(self, downloader=None):
1346 InfoExtractor.__init__(self, downloader)
1347
1348 @staticmethod
1349 def suitable(url):
1350 return (re.match(PhotobucketIE._VALID_URL, url) is not None)
1351
1352 def report_download_webpage(self, video_id):
1353 """Report webpage download."""
1354 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
1355
1356 def report_extraction(self, video_id):
1357 """Report information extraction."""
1358 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
1359
1360 def _real_initialize(self):
1361 return
1362
1363 def _real_extract(self, url):
1364 # Extract id from URL
1365 mobj = re.match(self._VALID_URL, url)
1366 if mobj is None:
1367 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1368 return
1369
1370 # At this point we have a new video
1371 self._downloader.increment_downloads()
1372 video_id = mobj.group(1)
1373
1374 video_extension = 'flv'
1375
1376 # Retrieve video webpage to extract further information
1377 request = urllib2.Request(url)
1378 try:
1379 self.report_download_webpage(video_id)
1380 webpage = urllib2.urlopen(request).read()
1381 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1382 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1383 return
1384
1385 # Extract URL, uploader, and title from webpage
1386 self.report_extraction(video_id)
1387 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1388 if mobj is None:
1389 self._downloader.trouble(u'ERROR: unable to extract media URL')
1390 return
1391 mediaURL = urllib.unquote(mobj.group(1))
1392
1393 video_url = mediaURL
1394
1395 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1396 if mobj is None:
1397 self._downloader.trouble(u'ERROR: unable to extract title')
1398 return
1399 video_title = mobj.group(1).decode('utf-8')
1400 video_title = sanitize_title(video_title)
1401 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1402
1403 video_uploader = mobj.group(2).decode('utf-8')
1404
1405 try:
1406 # Process video information
1407 self._downloader.process_info({
1408 'id': video_id.decode('utf-8'),
1409 'url': video_url.decode('utf-8'),
1410 'uploader': video_uploader,
1411 'upload_date': u'NA',
1412 'title': video_title,
1413 'stitle': simple_title,
1414 'ext': video_extension.decode('utf-8'),
1415 'format': u'NA',
1416 'player_url': None,
1417 })
1418 except UnavailableVideoError:
1419 self._downloader.trouble(u'ERROR: unable to download video')
1420
1421
1422 class YahooIE(InfoExtractor):
1423 """Information extractor for video.yahoo.com."""
1424
1425 # _VALID_URL matches all Yahoo! Video URLs
1426 # _VPAGE_URL matches only the extractable '/watch/' URLs
1427 _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1428 _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1429
1430 def __init__(self, downloader=None):
1431 InfoExtractor.__init__(self, downloader)
1432
1433 @staticmethod
1434 def suitable(url):
1435 return (re.match(YahooIE._VALID_URL, url) is not None)
1436
1437 def report_download_webpage(self, video_id):
1438 """Report webpage download."""
1439 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
1440
1441 def report_extraction(self, video_id):
1442 """Report information extraction."""
1443 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
1444
1445 def _real_initialize(self):
1446 return
1447
1448 def _real_extract(self, url, new_video=True):
1449 # Extract ID from URL
1450 mobj = re.match(self._VALID_URL, url)
1451 if mobj is None:
1452 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1453 return
1454
1455 # At this point we have a new video
1456 self._downloader.increment_downloads()
1457 video_id = mobj.group(2)
1458 video_extension = 'flv'
1459
1460 # Rewrite valid but non-extractable URLs as
1461 # extractable English language /watch/ URLs
1462 if re.match(self._VPAGE_URL, url) is None:
1463 request = urllib2.Request(url)
1464 try:
1465 webpage = urllib2.urlopen(request).read()
1466 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1467 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1468 return
1469
1470 mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1471 if mobj is None:
1472 self._downloader.trouble(u'ERROR: Unable to extract id field')
1473 return
1474 yahoo_id = mobj.group(1)
1475
1476 mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1477 if mobj is None:
1478 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1479 return
1480 yahoo_vid = mobj.group(1)
1481
1482 url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1483 return self._real_extract(url, new_video=False)
1484
1485 # Retrieve video webpage to extract further information
1486 request = urllib2.Request(url)
1487 try:
1488 self.report_download_webpage(video_id)
1489 webpage = urllib2.urlopen(request).read()
1490 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1491 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1492 return
1493
1494 # Extract uploader and title from webpage
1495 self.report_extraction(video_id)
1496 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1497 if mobj is None:
1498 self._downloader.trouble(u'ERROR: unable to extract video title')
1499 return
1500 video_title = mobj.group(1).decode('utf-8')
1501 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1502
1503 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1504 if mobj is None:
1505 self._downloader.trouble(u'ERROR: unable to extract video uploader')
1506 return
1507 video_uploader = mobj.group(1).decode('utf-8')
1508
1509 # Extract video thumbnail
1510 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1511 if mobj is None:
1512 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1513 return
1514 video_thumbnail = mobj.group(1).decode('utf-8')
1515
1516 # Extract video description
1517 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1518 if mobj is None:
1519 self._downloader.trouble(u'ERROR: unable to extract video description')
1520 return
1521 video_description = mobj.group(1).decode('utf-8')
1522 if not video_description: video_description = 'No description available.'
1523
1524 # Extract video height and width
1525 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1526 if mobj is None:
1527 self._downloader.trouble(u'ERROR: unable to extract video height')
1528 return
1529 yv_video_height = mobj.group(1)
1530
1531 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1532 if mobj is None:
1533 self._downloader.trouble(u'ERROR: unable to extract video width')
1534 return
1535 yv_video_width = mobj.group(1)
1536
1537 # Retrieve video playlist to extract media URL
1538 # I'm not completely sure what all these options are, but we
1539 # seem to need most of them, otherwise the server sends a 401.
1540 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
1541 yv_bitrate = '700' # according to Wikipedia this is hard-coded
1542 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1543 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1544 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1545 try:
1546 self.report_download_webpage(video_id)
1547 webpage = urllib2.urlopen(request).read()
1548 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1549 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1550 return
1551
1552 # Extract media URL from playlist XML
1553 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1554 if mobj is None:
1555 self._downloader.trouble(u'ERROR: Unable to extract media URL')
1556 return
1557 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1558 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1559
1560 try:
1561 # Process video information
1562 self._downloader.process_info({
1563 'id': video_id.decode('utf-8'),
1564 'url': video_url,
1565 'uploader': video_uploader,
1566 'upload_date': u'NA',
1567 'title': video_title,
1568 'stitle': simple_title,
1569 'ext': video_extension.decode('utf-8'),
1570 'thumbnail': video_thumbnail.decode('utf-8'),
1571 'description': video_description,
1572 'thumbnail': video_thumbnail,
1573 'description': video_description,
1574 'player_url': None,
1575 })
1576 except UnavailableVideoError:
1577 self._downloader.trouble(u'ERROR: unable to download video')
1578
1579
1580 class GenericIE(InfoExtractor):
1581 """Generic last-resort information extractor."""
1582
1583 def __init__(self, downloader=None):
1584 InfoExtractor.__init__(self, downloader)
1585
1586 @staticmethod
1587 def suitable(url):
1588 return True
1589
1590 def report_download_webpage(self, video_id):
1591 """Report webpage download."""
1592 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
1593 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
1594
1595 def report_extraction(self, video_id):
1596 """Report information extraction."""
1597 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
1598
1599 def _real_initialize(self):
1600 return
1601
1602 def _real_extract(self, url):
1603 # At this point we have a new video
1604 self._downloader.increment_downloads()
1605
1606 video_id = url.split('/')[-1]
1607 request = urllib2.Request(url)
1608 try:
1609 self.report_download_webpage(video_id)
1610 webpage = urllib2.urlopen(request).read()
1611 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1612 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1613 return
1614 except ValueError, err:
1615 # since this is the last-resort InfoExtractor, if
1616 # this error is thrown, it'll be thrown here
1617 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1618 return
1619
1620 # Start with something easy: JW Player in SWFObject
1621 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1622 if mobj is None:
1623 # Broaden the search a little bit
1624 mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1625 if mobj is None:
1626 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1627 return
1628
1629 # It's possible that one of the regexes
1630 # matched, but returned an empty group:
1631 if mobj.group(1) is None:
1632 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1633 return
1634
1635 video_url = urllib.unquote(mobj.group(1))
1636 video_id = os.path.basename(video_url)
1637
1638 # here's a fun little line of code for you:
1639 video_extension = os.path.splitext(video_id)[1][1:]
1640 video_id = os.path.splitext(video_id)[0]
1641
1642 # it's tempting to parse this further, but you would
1643 # have to take into account all the variations like
1644 # Video Title - Site Name
1645 # Site Name | Video Title
1646 # Video Title - Tagline | Site Name
1647 # and so on and so forth; it's just not practical
1648 mobj = re.search(r'<title>(.*)</title>', webpage)
1649 if mobj is None:
1650 self._downloader.trouble(u'ERROR: unable to extract title')
1651 return
1652 video_title = mobj.group(1).decode('utf-8')
1653 video_title = sanitize_title(video_title)
1654 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
1655
1656 # video uploader is domain name
1657 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1658 if mobj is None:
1659 self._downloader.trouble(u'ERROR: unable to extract title')
1660 return
1661 video_uploader = mobj.group(1).decode('utf-8')
1662
1663 try:
1664 # Process video information
1665 self._downloader.process_info({
1666 'id': video_id.decode('utf-8'),
1667 'url': video_url.decode('utf-8'),
1668 'uploader': video_uploader,
1669 'upload_date': u'NA',
1670 'title': video_title,
1671 'stitle': simple_title,
1672 'ext': video_extension.decode('utf-8'),
1673 'format': u'NA',
1674 'player_url': None,
1675 })
1676 except UnavailableVideoError, err:
1677 self._downloader.trouble(u'ERROR: unable to download video')
1678
1679
1680 class YoutubeSearchIE(InfoExtractor):
1681 """Information Extractor for YouTube search queries."""
1682 _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
1683 _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
1684 _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
1685 _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1686 _youtube_ie = None
1687 _max_youtube_results = 1000
1688
1689 def __init__(self, youtube_ie, downloader=None):
1690 InfoExtractor.__init__(self, downloader)
1691 self._youtube_ie = youtube_ie
1692
1693 @staticmethod
1694 def suitable(url):
1695 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
1696
1697 def report_download_page(self, query, pagenum):
1698 """Report attempt to download playlist page with given number."""
1699 query = query.decode(preferredencoding())
1700 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1701
1702 def _real_initialize(self):
1703 self._youtube_ie.initialize()
1704
1705 def _real_extract(self, query):
1706 mobj = re.match(self._VALID_QUERY, query)
1707 if mobj is None:
1708 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1709 return
1710
1711 prefix, query = query.split(':')
1712 prefix = prefix[8:]
1713 query = query.encode('utf-8')
1714 if prefix == '':
1715 self._download_n_results(query, 1)
1716 return
1717 elif prefix == 'all':
1718 self._download_n_results(query, self._max_youtube_results)
1719 return
1720 else:
1721 try:
1722 n = long(prefix)
1723 if n <= 0:
1724 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1725 return
1726 elif n > self._max_youtube_results:
1727 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
1728 n = self._max_youtube_results
1729 self._download_n_results(query, n)
1730 return
1731 except ValueError: # parsing prefix as integer fails
1732 self._download_n_results(query, 1)
1733 return
1734
1735 def _download_n_results(self, query, n):
1736 """Downloads a specified number of results for a query"""
1737
1738 video_ids = []
1739 already_seen = set()
1740 pagenum = 1
1741
1742 while True:
1743 self.report_download_page(query, pagenum)
1744 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1745 request = urllib2.Request(result_url, None, std_headers)
1746 try:
1747 page = urllib2.urlopen(request).read()
1748 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1749 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1750 return
1751
1752 # Extract video identifiers
1753 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1754 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
1755 if video_id not in already_seen:
1756 video_ids.append(video_id)
1757 already_seen.add(video_id)
1758 if len(video_ids) == n:
1759 # Specified n videos reached
1760 for id in video_ids:
1761 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1762 return
1763
1764 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1765 for id in video_ids:
1766 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1767 return
1768
1769 pagenum = pagenum + 1
1770
1771 class GoogleSearchIE(InfoExtractor):
1772 """Information Extractor for Google Video search queries."""
1773 _VALID_QUERY = r'gvsearch(\d+|all)?:[\s\S]+'
1774 _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1775 _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
1776 _MORE_PAGES_INDICATOR = r'<span>Next</span>'
1777 _google_ie = None
1778 _max_google_results = 1000
1779
1780 def __init__(self, google_ie, downloader=None):
1781 InfoExtractor.__init__(self, downloader)
1782 self._google_ie = google_ie
1783
1784 @staticmethod
1785 def suitable(url):
1786 return (re.match(GoogleSearchIE._VALID_QUERY, url) is not None)
1787
1788 def report_download_page(self, query, pagenum):
1789 """Report attempt to download playlist page with given number."""
1790 query = query.decode(preferredencoding())
1791 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1792
1793 def _real_initialize(self):
1794 self._google_ie.initialize()
1795
1796 def _real_extract(self, query):
1797 mobj = re.match(self._VALID_QUERY, query)
1798 if mobj is None:
1799 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1800 return
1801
1802 prefix, query = query.split(':')
1803 prefix = prefix[8:]
1804 query = query.encode('utf-8')
1805 if prefix == '':
1806 self._download_n_results(query, 1)
1807 return
1808 elif prefix == 'all':
1809 self._download_n_results(query, self._max_google_results)
1810 return
1811 else:
1812 try:
1813 n = long(prefix)
1814 if n <= 0:
1815 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1816 return
1817 elif n > self._max_google_results:
1818 self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
1819 n = self._max_google_results
1820 self._download_n_results(query, n)
1821 return
1822 except ValueError: # parsing prefix as integer fails
1823 self._download_n_results(query, 1)
1824 return
1825
1826 def _download_n_results(self, query, n):
1827 """Downloads a specified number of results for a query"""
1828
1829 video_ids = []
1830 already_seen = set()
1831 pagenum = 1
1832
1833 while True:
1834 self.report_download_page(query, pagenum)
1835 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1836 request = urllib2.Request(result_url, None, std_headers)
1837 try:
1838 page = urllib2.urlopen(request).read()
1839 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1840 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1841 return
1842
1843 # Extract video identifiers
1844 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1845 video_id = mobj.group(1)
1846 if video_id not in already_seen:
1847 video_ids.append(video_id)
1848 already_seen.add(video_id)
1849 if len(video_ids) == n:
1850 # Specified n videos reached
1851 for id in video_ids:
1852 self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1853 return
1854
1855 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1856 for id in video_ids:
1857 self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
1858 return
1859
1860 pagenum = pagenum + 1
1861
1862 class YahooSearchIE(InfoExtractor):
1863 """Information Extractor for Yahoo! Video search queries."""
1864 _VALID_QUERY = r'yvsearch(\d+|all)?:[\s\S]+'
1865 _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1866 _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1867 _MORE_PAGES_INDICATOR = r'\s*Next'
1868 _yahoo_ie = None
1869 _max_yahoo_results = 1000
1870
1871 def __init__(self, yahoo_ie, downloader=None):
1872 InfoExtractor.__init__(self, downloader)
1873 self._yahoo_ie = yahoo_ie
1874
1875 @staticmethod
1876 def suitable(url):
1877 return (re.match(YahooSearchIE._VALID_QUERY, url) is not None)
1878
1879 def report_download_page(self, query, pagenum):
1880 """Report attempt to download playlist page with given number."""
1881 query = query.decode(preferredencoding())
1882 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1883
1884 def _real_initialize(self):
1885 self._yahoo_ie.initialize()
1886
1887 def _real_extract(self, query):
1888 mobj = re.match(self._VALID_QUERY, query)
1889 if mobj is None:
1890 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1891 return
1892
1893 prefix, query = query.split(':')
1894 prefix = prefix[8:]
1895 query = query.encode('utf-8')
1896 if prefix == '':
1897 self._download_n_results(query, 1)
1898 return
1899 elif prefix == 'all':
1900 self._download_n_results(query, self._max_yahoo_results)
1901 return
1902 else:
1903 try:
1904 n = long(prefix)
1905 if n <= 0:
1906 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1907 return
1908 elif n > self._max_yahoo_results:
1909 self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
1910 n = self._max_yahoo_results
1911 self._download_n_results(query, n)
1912 return
1913 except ValueError: # parsing prefix as integer fails
1914 self._download_n_results(query, 1)
1915 return
1916
1917 def _download_n_results(self, query, n):
1918 """Downloads a specified number of results for a query"""
1919
1920 video_ids = []
1921 already_seen = set()
1922 pagenum = 1
1923
1924 while True:
1925 self.report_download_page(query, pagenum)
1926 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
1927 request = urllib2.Request(result_url, None, std_headers)
1928 try:
1929 page = urllib2.urlopen(request).read()
1930 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1931 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1932 return
1933
1934 # Extract video identifiers
1935 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1936 video_id = mobj.group(1)
1937 if video_id not in already_seen:
1938 video_ids.append(video_id)
1939 already_seen.add(video_id)
1940 if len(video_ids) == n:
1941 # Specified n videos reached
1942 for id in video_ids:
1943 self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1944 return
1945
1946 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1947 for id in video_ids:
1948 self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
1949 return
1950
1951 pagenum = pagenum + 1
1952
1953 class YoutubePlaylistIE(InfoExtractor):
1954 """Information Extractor for YouTube playlists."""
1955
1956 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*'
1957 _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
1958 _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
1959 _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
1960 _youtube_ie = None
1961
1962 def __init__(self, youtube_ie, downloader=None):
1963 InfoExtractor.__init__(self, downloader)
1964 self._youtube_ie = youtube_ie
1965
1966 @staticmethod
1967 def suitable(url):
1968 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
1969
1970 def report_download_page(self, playlist_id, pagenum):
1971 """Report attempt to download playlist page with given number."""
1972 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1973
1974 def _real_initialize(self):
1975 self._youtube_ie.initialize()
1976
1977 def _real_extract(self, url):
1978 # Extract playlist id
1979 mobj = re.match(self._VALID_URL, url)
1980 if mobj is None:
1981 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1982 return
1983
1984 # Download playlist pages
1985 playlist_id = mobj.group(1)
1986 video_ids = []
1987 pagenum = 1
1988
1989 while True:
1990 self.report_download_page(playlist_id, pagenum)
1991 request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1992 try:
1993 page = urllib2.urlopen(request).read()
1994 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1995 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1996 return
1997
1998 # Extract video identifiers
1999 ids_in_page = []
2000 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2001 if mobj.group(1) not in ids_in_page:
2002 ids_in_page.append(mobj.group(1))
2003 video_ids.extend(ids_in_page)
2004
2005 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2006 break
2007 pagenum = pagenum + 1
2008
2009 playliststart = self._downloader.params.get('playliststart', 1) - 1
2010 playlistend = self._downloader.params.get('playlistend', -1)
2011 video_ids = video_ids[playliststart:playlistend]
2012
2013 for id in video_ids:
2014 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2015 return
2016
2017 class YoutubeUserIE(InfoExtractor):
2018 """Information Extractor for YouTube users."""
2019
2020 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/user/(.*)'
2021 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
2022 _VIDEO_INDICATOR = r'http://gdata.youtube.com/feeds/api/videos/(.*)' # XXX Fix this.
2023 _youtube_ie = None
2024
2025 def __init__(self, youtube_ie, downloader=None):
2026 InfoExtractor.__init__(self, downloader)
2027 self._youtube_ie = youtube_ie
2028
2029 @staticmethod
2030 def suitable(url):
2031 return (re.match(YoutubeUserIE._VALID_URL, url) is not None)
2032
2033 def report_download_page(self, username):
2034 """Report attempt to download user page."""
2035 self._downloader.to_screen(u'[youtube] user %s: Downloading page ' % (username))
2036
2037 def _real_initialize(self):
2038 self._youtube_ie.initialize()
2039
2040 def _real_extract(self, url):
2041 # Extract username
2042 mobj = re.match(self._VALID_URL, url)
2043 if mobj is None:
2044 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2045 return
2046
2047 # Download user page
2048 username = mobj.group(1)
2049 video_ids = []
2050 pagenum = 1
2051
2052 self.report_download_page(username)
2053 request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
2054 try:
2055 page = urllib2.urlopen(request).read()
2056 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2057 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2058 return
2059
2060 # Extract video identifiers
2061 ids_in_page = []
2062
2063 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2064 if mobj.group(1) not in ids_in_page:
2065 ids_in_page.append(mobj.group(1))
2066 video_ids.extend(ids_in_page)
2067
2068 playliststart = self._downloader.params.get('playliststart', 1) - 1
2069 playlistend = self._downloader.params.get('playlistend', -1)
2070 video_ids = video_ids[playliststart:playlistend]
2071
2072 for id in video_ids:
2073 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2074 return
2075
2076 class PostProcessor(object):
2077 """Post Processor class.
2078
2079 PostProcessor objects can be added to downloaders with their
2080 add_post_processor() method. When the downloader has finished a
2081 successful download, it will take its internal chain of PostProcessors
2082 and start calling the run() method on each one of them, first with
2083 an initial argument and then with the returned value of the previous
2084 PostProcessor.
2085
2086 The chain will be stopped if one of them ever returns None or the end
2087 of the chain is reached.
2088
2089 PostProcessor objects follow a "mutual registration" process similar
2090 to InfoExtractor objects.
2091 """
2092
2093 _downloader = None
2094
2095 def __init__(self, downloader=None):
2096 self._downloader = downloader
2097
2098 def set_downloader(self, downloader):
2099 """Sets the downloader for this PP."""
2100 self._downloader = downloader
2101
2102 def run(self, information):
2103 """Run the PostProcessor.
2104
2105 The "information" argument is a dictionary like the ones
2106 composed by InfoExtractors. The only difference is that this
2107 one has an extra field called "filepath" that points to the
2108 downloaded file.
2109
2110 When this method returns None, the postprocessing chain is
2111 stopped. However, this method may return an information
2112 dictionary that will be passed to the next postprocessing
2113 object in the chain. It can be the one it received after
2114 changing some fields.
2115
2116 In addition, this method may raise a PostProcessingError
2117 exception that will be taken into account by the downloader
2118 it was called from.
2119 """
2120 return information # by default, do nothing
2121
2122 ### MAIN PROGRAM ###
2123 if __name__ == '__main__':
2124 try:
2125 # Modules needed only when running the main program
2126 import getpass
2127 import optparse
2128
2129 # Function to update the program file with the latest version from bitbucket.org
2130 def update_self(downloader, filename):
2131 # Note: downloader only used for options
2132 if not os.access (filename, os.W_OK):
2133 sys.exit('ERROR: no write permissions on %s' % filename)
2134
2135 downloader.to_screen('Updating to latest stable version...')
2136 latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
2137 latest_version = urllib.urlopen(latest_url).read().strip()
2138 prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
2139 newcontent = urllib.urlopen(prog_url).read()
2140 stream = open(filename, 'w')
2141 stream.write(newcontent)
2142 stream.close()
2143 downloader.to_screen('Updated to version %s' % latest_version)
2144
2145 # Parse command line
2146 parser = optparse.OptionParser(
2147 usage='Usage: %prog [options] url...',
2148 version='2010.11.19',
2149 conflict_handler='resolve',
2150 )
2151
2152 parser.add_option('-h', '--help',
2153 action='help', help='print this help text and exit')
2154 parser.add_option('-v', '--version',
2155 action='version', help='print program version and exit')
2156 parser.add_option('-U', '--update',
2157 action='store_true', dest='update_self', help='update this program to latest stable version')
2158 parser.add_option('-i', '--ignore-errors',
2159 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
2160 parser.add_option('-r', '--rate-limit',
2161 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
2162 parser.add_option('-R', '--retries',
2163 dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
2164 parser.add_option('--playlist-start',
2165 dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
2166 parser.add_option('--playlist-end',
2167 dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
2168
2169 authentication = optparse.OptionGroup(parser, 'Authentication Options')
2170 authentication.add_option('-u', '--username',
2171 dest='username', metavar='USERNAME', help='account username')
2172 authentication.add_option('-p', '--password',
2173 dest='password', metavar='PASSWORD', help='account password')
2174 authentication.add_option('-n', '--netrc',
2175 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
2176 parser.add_option_group(authentication)
2177
2178 video_format = optparse.OptionGroup(parser, 'Video Format Options')
2179 video_format.add_option('-f', '--format',
2180 action='store', dest='format', metavar='FORMAT', help='video format code')
2181 video_format.add_option('-m', '--mobile-version',
2182 action='store_const', dest='format', help='alias for -f 17', const='17')
2183 video_format.add_option('--all-formats',
2184 action='store_const', dest='format', help='download all available video formats', const='-1')
2185 video_format.add_option('--max-quality',
2186 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
2187 video_format.add_option('-b', '--best-quality',
2188 action='store_true', dest='bestquality', help='download the best video quality (DEPRECATED)')
2189 parser.add_option_group(video_format)
2190
2191 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
2192 verbosity.add_option('-q', '--quiet',
2193 action='store_true', dest='quiet', help='activates quiet mode', default=False)
2194 verbosity.add_option('-s', '--simulate',
2195 action='store_true', dest='simulate', help='do not download video', default=False)
2196 verbosity.add_option('-g', '--get-url',
2197 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
2198 verbosity.add_option('-e', '--get-title',
2199 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
2200 verbosity.add_option('--get-thumbnail',
2201 action='store_true', dest='getthumbnail', help='simulate, quiet but print thumbnail URL', default=False)
2202 verbosity.add_option('--get-description',
2203 action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False)
2204 verbosity.add_option('--no-progress',
2205 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
2206 parser.add_option_group(verbosity)
2207
2208 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
2209 filesystem.add_option('-t', '--title',
2210 action='store_true', dest='usetitle', help='use title in file name', default=False)
2211 filesystem.add_option('-l', '--literal',
2212 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
2213 filesystem.add_option('-A', '--auto-number',
2214 action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False)
2215 filesystem.add_option('-o', '--output',
2216 dest='outtmpl', metavar='TEMPLATE', help='output filename template')
2217 filesystem.add_option('-a', '--batch-file',
2218 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
2219 filesystem.add_option('-w', '--no-overwrites',
2220 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
2221 filesystem.add_option('-c', '--continue',
2222 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
2223 filesystem.add_option('--cookies',
2224 dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
2225 parser.add_option_group(filesystem)
2226
2227 (opts, args) = parser.parse_args()
2228
2229 # Open appropriate CookieJar
2230 if opts.cookiefile is None:
2231 jar = cookielib.CookieJar()
2232 else:
2233 try:
2234 jar = cookielib.MozillaCookieJar(opts.cookiefile)
2235 if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
2236 jar.load()
2237 except (IOError, OSError), err:
2238 sys.exit(u'ERROR: unable to open cookie file')
2239
2240 # General configuration
2241 cookie_processor = urllib2.HTTPCookieProcessor(jar)
2242 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
2243 urllib2.install_opener(urllib2.build_opener(cookie_processor))
2244 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
2245
2246 # Batch file verification
2247 batchurls = []
2248 if opts.batchfile is not None:
2249 try:
2250 if opts.batchfile == '-':
2251 batchfd = sys.stdin
2252 else:
2253 batchfd = open(opts.batchfile, 'r')
2254 batchurls = batchfd.readlines()
2255 batchurls = [x.strip() for x in batchurls]
2256 batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
2257 except IOError:
2258 sys.exit(u'ERROR: batch file could not be read')
2259 all_urls = batchurls + args
2260
2261 # Conflicting, missing and erroneous options
2262 if opts.bestquality:
2263 print >>sys.stderr, u'\nWARNING: -b/--best-quality IS DEPRECATED AS IT IS THE DEFAULT BEHAVIOR NOW\n'
2264 if opts.usenetrc and (opts.username is not None or opts.password is not None):
2265 parser.error(u'using .netrc conflicts with giving username/password')
2266 if opts.password is not None and opts.username is None:
2267 parser.error(u'account username missing')
2268 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber):
2269 parser.error(u'using output template conflicts with using title, literal title or auto number')
2270 if opts.usetitle and opts.useliteral:
2271 parser.error(u'using title conflicts with using literal title')
2272 if opts.username is not None and opts.password is None:
2273 opts.password = getpass.getpass(u'Type account password and press return:')
2274 if opts.ratelimit is not None:
2275 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
2276 if numeric_limit is None:
2277 parser.error(u'invalid rate limit specified')
2278 opts.ratelimit = numeric_limit
2279 if opts.retries is not None:
2280 try:
2281 opts.retries = long(opts.retries)
2282 except (TypeError, ValueError), err:
2283 parser.error(u'invalid retry count specified')
2284 try:
2285 opts.playliststart = long(opts.playliststart)
2286 if opts.playliststart <= 0:
2287 raise ValueError
2288 except (TypeError, ValueError), err:
2289 parser.error(u'invalid playlist start number specified')
2290 try:
2291 opts.playlistend = long(opts.playlistend)
2292 if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
2293 raise ValueError
2294 except (TypeError, ValueError), err:
2295 parser.error(u'invalid playlist end number specified')
2296
2297 # Information extractors
2298 youtube_ie = YoutubeIE()
2299 metacafe_ie = MetacafeIE(youtube_ie)
2300 dailymotion_ie = DailymotionIE()
2301 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
2302 youtube_user_ie = YoutubeUserIE(youtube_ie)
2303 youtube_search_ie = YoutubeSearchIE(youtube_ie)
2304 google_ie = GoogleIE()
2305 google_search_ie = GoogleSearchIE(google_ie)
2306 photobucket_ie = PhotobucketIE()
2307 yahoo_ie = YahooIE()
2308 yahoo_search_ie = YahooSearchIE(yahoo_ie)
2309 generic_ie = GenericIE()
2310
2311 # File downloader
2312 fd = FileDownloader({
2313 'usenetrc': opts.usenetrc,
2314 'username': opts.username,
2315 'password': opts.password,
2316 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2317 'forceurl': opts.geturl,
2318 'forcetitle': opts.gettitle,
2319 'forcethumbnail': opts.getthumbnail,
2320 'forcedescription': opts.getdescription,
2321 'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
2322 'format': opts.format,
2323 'format_limit': opts.format_limit,
2324 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
2325 or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
2326 or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
2327 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
2328 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s')
2329 or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
2330 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
2331 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
2332 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
2333 or u'%(id)s.%(ext)s'),
2334 'ignoreerrors': opts.ignoreerrors,
2335 'ratelimit': opts.ratelimit,
2336 'nooverwrites': opts.nooverwrites,
2337 'retries': opts.retries,
2338 'continuedl': opts.continue_dl,
2339 'noprogress': opts.noprogress,
2340 'playliststart': opts.playliststart,
2341 'playlistend': opts.playlistend,
2342 'logtostderr': opts.outtmpl == '-',
2343 })
2344 fd.add_info_extractor(youtube_search_ie)
2345 fd.add_info_extractor(youtube_pl_ie)
2346 fd.add_info_extractor(youtube_user_ie)
2347 fd.add_info_extractor(metacafe_ie)
2348 fd.add_info_extractor(dailymotion_ie)
2349 fd.add_info_extractor(youtube_ie)
2350 fd.add_info_extractor(google_ie)
2351 fd.add_info_extractor(google_search_ie)
2352 fd.add_info_extractor(photobucket_ie)
2353 fd.add_info_extractor(yahoo_ie)
2354 fd.add_info_extractor(yahoo_search_ie)
2355
2356 # This must come last since it's the
2357 # fallback if none of the others work
2358 fd.add_info_extractor(generic_ie)
2359
2360 # Update version
2361 if opts.update_self:
2362 update_self(fd, sys.argv[0])
2363
2364 # Maybe do nothing
2365 if len(all_urls) < 1:
2366 if not opts.update_self:
2367 parser.error(u'you must provide at least one URL')
2368 else:
2369 sys.exit()
2370 retcode = fd.download(all_urls)
2371
2372 # Dump cookie jar if requested
2373 if opts.cookiefile is not None:
2374 try:
2375 jar.save()
2376 except (IOError, OSError), err:
2377 sys.exit(u'ERROR: unable to save cookie jar')
2378
2379 sys.exit(retcode)
2380
2381 except DownloadError:
2382 sys.exit(1)
2383 except SameFileError:
2384 sys.exit(u'ERROR: fixed output name but more than one file to download')
2385 except KeyboardInterrupt:
2386 sys.exit(u'\nERROR: Interrupted by user')