Imported Upstream version 2011.10.19

diff --git a/LATEST_VERSION b/LATEST_VERSION
index bae3956df68a933bbb69621a676c86f15de56b39..a68f991608e9aa3aa380f503d6f4fdd0848735df 100644 (file)
--- a/LATEST_VERSION
+++ b/LATEST_VERSION
@@ -1 +1 @@
-2011.09.27
+2011.10.19

diff --git a/README.md b/README.md
index ad0590f9983ba5890c28dd1d64881319e7b7ff11..2b0ab67eba2215b3c4d1f9189d1f2456ee03dad0 100644 (file)
--- a/README.md
+++ b/README.md
@@ -68,6 +68,7 @@ which means you can modify it, redistribute it or use it however you like.
     -f, --format FORMAT      video format code
     --all-formats            download all available video formats
     --max-quality FORMAT     highest quality format to download
     -f, --format FORMAT      video format code
     --all-formats            download all available video formats
     --max-quality FORMAT     highest quality format to download

+    -F, --list-formats       list all available formats (currently youtube only)

 
 ### Authentication Options:
     -u, --username USERNAME  account username
 
 ### Authentication Options:
     -u, --username USERNAME  account username

diff --git a/youtube-dl b/youtube-dl
index 6cb58f1a8c886c38b71135c81ea17881218ca27f..3a37fae2eb19769886f897090dbd28e397ee3d21 100755 (executable)
--- a/youtube-dl
+++ b/youtube-dl
@@ -15,7 +15,7 @@ __author__  = (
        )
 
 __license__ = 'Public Domain'
        )
 
 __license__ = 'Public Domain'

-__version__ = '2011.09.27'
+__version__ = '2011.10.19'

 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 
 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 


@@ -1103,6 +1103,21 @@ class YoutubeIE(InfoExtractor):
                '44': 'webm',
                '45': 'webm',
        }
                '44': 'webm',
                '45': 'webm',
        }

+       _video_dimensions = {
+               '5': '240x400',
+               '6': '???',
+               '13': '???',
+               '17': '144x176',
+               '18': '360x640',
+               '22': '720x1280',
+               '34': '360x640',
+               '35': '480x854',
+               '37': '1080x1920',
+               '38': '3072x4096',
+               '43': '360x640',
+               '44': '480x854',
+               '45': '720x1280',
+       }       

        IE_NAME = u'youtube'
 
        def report_lang(self):
        IE_NAME = u'youtube'
 
        def report_lang(self):


@@ -1137,6 +1152,11 @@ class YoutubeIE(InfoExtractor):
                """Indicate the download will use the RTMP protocol."""
                self._downloader.to_screen(u'[youtube] RTMP download detected')
 
                """Indicate the download will use the RTMP protocol."""
                self._downloader.to_screen(u'[youtube] RTMP download detected')
 

+       def _print_formats(self, formats):
+               print 'Available formats:'
+               for x in formats:
+                       print '%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))
+

        def _real_initialize(self):
                if self._downloader is None:
                        return
        def _real_initialize(self):
                if self._downloader is None:
                        return


@@ -1216,7 +1236,7 @@ class YoutubeIE(InfoExtractor):
 
                # Get video webpage
                self.report_video_webpage_download(video_id)
 
                # Get video webpage
                self.report_video_webpage_download(video_id)

-               request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
+               request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)

                try:
                        video_webpage = urllib2.urlopen(request).read()
                except (urllib2.URLError, httplib.HTTPException, socket.error), err:
                try:
                        video_webpage = urllib2.urlopen(request).read()
                except (urllib2.URLError, httplib.HTTPException, socket.error), err:


@@ -1330,6 +1350,9 @@ class YoutubeIE(InfoExtractor):
                        if len(existing_formats) == 0:
                                self._downloader.trouble(u'ERROR: no known formats available for video')
                                return
                        if len(existing_formats) == 0:
                                self._downloader.trouble(u'ERROR: no known formats available for video')
                                return

+                       if self._downloader.params.get('listformats', None):
+                               self._print_formats(existing_formats)
+                               return

                        if req_format is None or req_format == 'best':
                                video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
                        elif req_format == 'worst':
                        if req_format is None or req_format == 'best':
                                video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
                        elif req_format == 'worst':


@@ -2036,6 +2059,18 @@ class VimeoIE(InfoExtractor):
                        return
                sig = mobj.group(1).decode('utf-8')
 
                        return
                sig = mobj.group(1).decode('utf-8')
 

+               # Vimeo specific: extract video quality information
+               mobj = re.search(r'<isHD>(\d+)</isHD>', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video quality information')
+                       return
+               quality = mobj.group(1).decode('utf-8')
+
+               if int(quality) == 1:
+                       quality = 'hd'
+               else:
+                       quality = 'sd'
+

                # Vimeo specific: Extract request signature expiration
                mobj = re.search(r'<request_signature_expires>(.*?)</request_signature_expires>', webpage)
                if mobj is None:
                # Vimeo specific: Extract request signature expiration
                mobj = re.search(r'<request_signature_expires>(.*?)</request_signature_expires>', webpage)
                if mobj is None:


@@ -2043,7 +2078,7 @@ class VimeoIE(InfoExtractor):
                        return
                sig_exp = mobj.group(1).decode('utf-8')
 
                        return
                sig_exp = mobj.group(1).decode('utf-8')
 

-               video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s" % (video_id, sig, sig_exp)
+               video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s/?q=%s" % (video_id, sig, sig_exp, quality)

 
                try:
                        # Process video information
 
                try:
                        # Process video information


@@ -2435,7 +2470,7 @@ class YahooSearchIE(InfoExtractor):
 class YoutubePlaylistIE(InfoExtractor):
        """Information Extractor for YouTube playlists."""
 
 class YoutubePlaylistIE(InfoExtractor):
        """Information Extractor for YouTube playlists."""
 

-       _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+       _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'

        _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
        _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
        _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
        _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
        _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
        _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'


@@ -3283,6 +3318,168 @@ class EscapistIE(InfoExtractor):
                        self._downloader.trouble(u'\nERROR: unable to download ' + videoId)
 
 
                        self._downloader.trouble(u'\nERROR: unable to download ' + videoId)
 
 

+class CollegeHumorIE(InfoExtractor):
+       """Information extractor for collegehumor.com"""
+
+       _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
+       IE_NAME = u'collegehumor'
+
+       def report_webpage(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+       def _simplify_title(self, title):
+               res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+               res = res.strip(ur'_')
+               return res
+
+       def _real_extract(self, url):
+               htmlParser = HTMLParser.HTMLParser()
+
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+               video_id = mobj.group('videoid')
+
+               self.report_webpage(video_id)
+               request = urllib2.Request(url)
+               try:
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+                       return
+
+               m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
+               if m is None:
+                       self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
+                       return
+               internal_video_id = m.group('internalvideoid')
+
+               info = {
+                       'id': video_id,
+                       'internal_id': internal_video_id,
+               }
+
+               self.report_extraction(video_id)
+               xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
+               try:
+                       metaXml = urllib2.urlopen(xmlUrl).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err))
+                       return
+
+               mdoc = xml.etree.ElementTree.fromstring(metaXml)
+               try:
+                       videoNode = mdoc.findall('./video')[0]
+                       info['description'] = videoNode.findall('./description')[0].text
+                       info['title'] = videoNode.findall('./caption')[0].text
+                       info['stitle'] = self._simplify_title(info['title'])
+                       info['url'] = videoNode.findall('./file')[0].text
+                       info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
+                       info['ext'] = info['url'].rpartition('.')[2]
+                       info['format'] = info['ext']
+               except IndexError:
+                       self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
+                       return
+
+               self._downloader.increment_downloads()
+
+               try:
+                       self._downloader.process_info(info)
+               except UnavailableVideoError, err:
+                       self._downloader.trouble(u'\nERROR: unable to download video')
+
+
+class XVideosIE(InfoExtractor):
+       """Information extractor for xvideos.com"""
+
+       _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
+       IE_NAME = u'xvideos'
+
+       def report_webpage(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+       def report_extraction(self, video_id):
+               """Report information extraction."""
+               self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+       def _simplify_title(self, title):
+               res = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', title)
+               res = res.strip(ur'_')
+               return res
+
+       def _real_extract(self, url):
+               htmlParser = HTMLParser.HTMLParser()
+
+               mobj = re.match(self._VALID_URL, url)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+                       return
+               video_id = mobj.group(1).decode('utf-8')
+
+               self.report_webpage(video_id)
+
+               request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
+               try:
+                       webpage = urllib2.urlopen(request).read()
+               except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+                       self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+                       return
+
+               self.report_extraction(video_id)
+
+
+               # Extract video URL
+               mobj = re.search(r'flv_url=(.+?)&', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video url')
+                       return
+               video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
+
+
+               # Extract title
+               mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video title')
+                       return
+               video_title = mobj.group(1).decode('utf-8')
+
+
+               # Extract video thumbnail
+               mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]/[a-fA-F0-9]/[a-fA-F0-9]/([a-fA-F0-9.]+jpg)', webpage)
+               if mobj is None:
+                       self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
+                       return
+               video_thumbnail = mobj.group(1).decode('utf-8')
+
+
+
+               self._downloader.increment_downloads()
+               info = {
+                       'id': video_id,
+                       'url': video_url,
+                       'uploader': None,
+                       'upload_date': None,
+                       'title': video_title,
+                       'stitle': self._simplify_title(video_title),
+                       'ext': 'flv',
+                       'format': 'flv',
+                       'thumbnail': video_thumbnail,
+                       'description': None,
+                       'player_url': None,
+               }
+
+               try:
+                       self._downloader.process_info(info)
+               except UnavailableVideoError, err:
+                       self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
+

 
 class PostProcessor(object):
        """Post Processor class.
 
 class PostProcessor(object):
        """Post Processor class.


@@ -3564,6 +3761,8 @@ def parseOpts():
                        action='store_const', dest='format', help='download all available video formats', const='all')
        video_format.add_option('--max-quality',
                        action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
                        action='store_const', dest='format', help='download all available video formats', const='all')
        video_format.add_option('--max-quality',
                        action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')

+       video_format.add_option('-F', '--list-formats',
+                       action='store_true', dest='listformats', help='list all available formats (currently youtube only)')

 
 
        verbosity.add_option('-q', '--quiet',
 
 
        verbosity.add_option('-q', '--quiet',


@@ -3676,6 +3875,8 @@ def gen_extractors():
                MyVideoIE(),
                ComedyCentralIE(),
                EscapistIE(),
                MyVideoIE(),
                ComedyCentralIE(),
                EscapistIE(),

+               CollegeHumorIE(),
+               XVideosIE(),

 
                GenericIE()
        ]
 
                GenericIE()
        ]


@@ -3784,6 +3985,7 @@ def main():
                'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
                'format': opts.format,
                'format_limit': opts.format_limit,
                'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
                'format': opts.format,
                'format_limit': opts.format_limit,

+               'listformats': opts.listformats,

                'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
                        or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
                        or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
                'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
                        or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
                        or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')