Raphaël G. Git Repositories - youtubedl/blob - youtube-dl

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2006 Ricardo Garcia Gonzalez
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining a
   6 # copy of this software and associated documentation files (the "Software"),
   7 # to deal in the Software without restriction, including without limitation
   8 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 # and/or sell copies of the Software, and to permit persons to whom the
  10 # Software is furnished to do so, subject to the following conditions:
  11 #
  12 # The above copyright notice and this permission notice shall be included
  13 # in all copies or substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21 # OTHER DEALINGS IN THE SOFTWARE.
  22 #
  23 # Except as contained in this notice, the name(s) of the above copyright
  24 # holders shall not be used in advertising or otherwise to promote the
  25 # sale, use or other dealings in this Software without prior written
  26 # authorization.
  27 #
  28 import sys
  29 import optparse
  30 import httplib
  31 import urllib2
  32 import re
  33 import string
  34 import os
  35 import time
  36 import netrc
  37 import getpass
  38
  39 # First off, check Python and refuse to run
  40 if sys.hexversion < 0x020400f0:
  41         sys.exit('Error: Python 2.4 or later needed to run the program')
  42
  43 # Global constants
  44 const_video_url_str = 'http://www.youtube.com/watch?v=%s'
  45 const_video_url_re = re.compile(r'(?:http://)?(?:www\d*\.)?youtube\.com/(?:v/|(?:watch(?:\.php)?)?\?v=)([^&]+).*')
  46 const_login_url_str = 'http://www.youtube.com/login?next=/watch%%3Fv%%3D%s'
  47 const_login_post_str = 'current_form=loginForm&next=%%2Fwatch%%3Fv%%3D%s&username=%s&password=%s&action_login=Log+In'
  48 const_age_url_str = 'http://www.youtube.com/verify_age?next_url=/watch%%3Fv%%3D%s'
  49 const_age_post_str = 'next_url=%%2Fwatch%%3Fv%%3D%s&action_confirm=Confirm'
  50 const_video_url_params_re = re.compile(r'player2\.swf\?([^"]+)"', re.M)
  51 const_video_url_real_str = 'http://www.youtube.com/get_video?%s'
  52 const_video_title_re = re.compile(r'<title>YouTube - ([^<]*)</title>', re.M | re.I)
  53 const_1k = 1024
  54 const_initial_block_size = 10 * const_1k
  55
  56 # Print error message, followed by standard advice information, and then exit
  57 def error_advice_exit(error_text):
  58         sys.stderr.write('Error: %s.\n' % error_text)
  59         sys.stderr.write('Try again several times. It may be a temporal problem.\n')
  60         sys.stderr.write('Other typical problems:\n\n')
  61         sys.stderr.write('\tVideo no longer exists.\n')
  62         sys.stderr.write('\tVideo requires age confirmation but you did not provide an account.\n')
  63         sys.stderr.write('\tYou provided the account data, but it is not valid.\n')
  64         sys.stderr.write('\tThe connection was cut suddenly for some reason.\n')
  65         sys.stderr.write('\tYouTube changed their system, and the program no longer works.\n')
  66         sys.stderr.write('\nTry to confirm you are able to view the video using a web browser.\n')
  67         sys.stderr.write('Use the same video URL and account information, if needed, with this program.\n')
  68         sys.stderr.write('When using a proxy, make sure http_proxy has http://host:port format.\n')
  69         sys.stderr.write('Try again several times and contact me if the problem persists.\n')
  70         sys.exit('\n')
  71
  72 # Wrapper to create custom requests with typical headers
  73 def request_create(url, data=None):
  74         retval = urllib2.Request(url)
  75         if data is not None:
  76                 retval.add_data(data)
  77         # Try to mimic Firefox, at least a little bit
  78         retval.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1) Gecko/20061010 Firefox/2.0')
  79         retval.add_header('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')
  80         retval.add_header('Accept', 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5')
  81         retval.add_header('Accept-Language', 'en-us,en;q=0.5')
  82         return retval
  83
  84 # Perform a request, process headers and return response
  85 def perform_request(url, data=None):
  86         request = request_create(url, data)
  87         response = urllib2.urlopen(request)
  88         return response
  89
  90 # Convert bytes to KiB
  91 def to_k(bytes):
  92         global const_1k
  93         return bytes / const_1k
  94
  95 # Conditional print
  96 def cond_print(str):
  97         global cmdl_opts
  98         if not cmdl_opts.quiet:
  99                 sys.stdout.write(str)
 100                 sys.stdout.flush()
 101
 102 # Title string normalization
 103 def title_string_norm(title):
 104         title = ''.join((x in string.ascii_letters or x in string.digits) and x or ' ' for x in title)
 105         title = '_'.join(title.split())
 106         title = title.lower()
 107         return title
 108
 109 # Generic download step
 110 def download_step(return_data_flag, step_title, step_error, url, post_data=None):
 111         try:
 112                 cond_print('%s... ' % step_title)
 113                 data = perform_request(url, post_data).read()
 114                 cond_print('done.\n')
 115                 if return_data_flag:
 116                         return data
 117                 return None
 118
 119         except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError):
 120                 cond_print('failed.\n')
 121                 error_advice_exit(step_error)
 122
 123         except KeyboardInterrupt:
 124                 sys.exit('\n')
 125
 126 # Generic extract step
 127 def extract_step(step_title, step_error, regexp, data):
 128         try:
 129                 cond_print('%s... ' % step_title)
 130                 match = regexp.search(data)
 131
 132                 if match is None:
 133                         cond_print('failed.\n')
 134                         error_advice_exit(step_error)
 135
 136                 extracted_data = match.group(1)
 137                 cond_print('done.\n')
 138                 return extracted_data
 139
 140         except KeyboardInterrupt:
 141                 sys.exit('\n')
 142
 143 # Calculate new block size based on previous block size
 144 def new_block_size(before, after, bytes):
 145         new_min = max(bytes / 2, 1)
 146         new_max = max(bytes * 2, 1)
 147         dif = after - before
 148         if dif < 0.0001:
 149                 return new_max
 150         rate = int(bytes / dif)
 151         if rate > new_max:
 152                 return new_max
 153         if rate < new_min:
 154                 return new_min
 155         return rate
 156
 157 # Create the command line options parser and parse command line
 158 cmdl_usage = 'usage: %prog [options] video_url'
 159 cmdl_version = '2007.01.19'
 160 cmdl_parser = optparse.OptionParser(usage=cmdl_usage, version=cmdl_version, conflict_handler='resolve')
 161 cmdl_parser.add_option('-h', '--help', action='help', help='print this help text and exit')
 162 cmdl_parser.add_option('-v', '--version', action='version', help='print program version and exit')
 163 cmdl_parser.add_option('-u', '--username', dest='username', metavar='USERNAME', help='account username')
 164 cmdl_parser.add_option('-p', '--password', dest='password', metavar='PASSWORD', help='account password')
 165 cmdl_parser.add_option('-o', '--output', dest='outfile', metavar='FILE', help='output video file name')
 166 cmdl_parser.add_option('-q', '--quiet', action='store_true', dest='quiet', help='activates quiet mode')
 167 cmdl_parser.add_option('-s', '--simulate', action='store_true', dest='simulate', help='do not download video')
 168 cmdl_parser.add_option('-t', '--title', action='store_true', dest='use_title', help='use title in file name')
 169 cmdl_parser.add_option('-n', '--netrc', action='store_true', dest='use_netrc', help='use .netrc authentication data')
 170 (cmdl_opts, cmdl_args) = cmdl_parser.parse_args()
 171
 172 # Get video URL
 173 if len(cmdl_args) != 1:
 174         cmdl_parser.print_help()
 175         sys.exit('\n')
 176 video_url_cmdl = cmdl_args[0]
 177
 178 # Verify video URL format and convert to "standard" format
 179 video_url_mo = const_video_url_re.match(video_url_cmdl)
 180 if video_url_mo is None:
 181         sys.exit('Error: URL does not seem to be a youtube video URL. If it is, report a bug.')
 182 video_url_id = video_url_mo.group(1)
 183 video_url = const_video_url_str % video_url_id
 184
 185 # Check conflicting options
 186 if cmdl_opts.outfile is not None and cmdl_opts.simulate:
 187         sys.stderr.write('Warning: video file name given but will not be used.\n')
 188
 189 if cmdl_opts.outfile is not None and cmdl_opts.use_title:
 190         sys.exit('Error: using the video title conflicts with using a given file name.')
 191
 192 if cmdl_opts.use_netrc and cmdl_opts.password is not None:
 193         sys.exit('Error: using netrc conflicts with giving command line password.')
 194
 195 # Incorrect option formatting
 196 if cmdl_opts.username is None and cmdl_opts.password is not None:
 197         sys.exit('Error: password give but username is missing.')
 198
 199 # Get account information if any
 200 account_username = None
 201 account_password = None
 202
 203 if cmdl_opts.use_netrc:
 204         try:
 205                 info = netrc.netrc().authenticators('youtube')
 206                 if info is None:
 207                         sys.exit('Error: no authenticators for machine youtube.')
 208                 netrc_username = info[0]
 209                 netrc_password = info[2]
 210         except IOError:
 211                 sys.exit('Error: unable to read .netrc file.')
 212         except netrc.NetrcParseError:
 213                 sys.exit('Error: unable to parse .netrc file.')
 214
 215 if cmdl_opts.password is not None:
 216         account_username = cmdl_opts.username
 217         account_password = cmdl_opts.password
 218 else:
 219         if cmdl_opts.username is not None and cmdl_opts.use_netrc:
 220                 if cmdl_opts.username != netrc_username:
 221                         sys.exit('Error: conflicting username from .netrc and command line options.')
 222                 account_username = cmdl_opts.username
 223                 account_password = netrc_password
 224         elif cmdl_opts.username is not None:
 225                 account_username = cmdl_opts.username
 226                 account_password = getpass.getpass('Type YouTube password and press return: ')
 227         elif cmdl_opts.use_netrc:
 228                 if len(netrc_username) == 0:
 229                         sys.exit('Error: empty username in .netrc file.')
 230                 account_username = netrc_username
 231                 account_password = netrc_password
 232
 233 # Get output file name
 234 if cmdl_opts.outfile is None:
 235         video_filename = '%s.flv' % video_url_id
 236 else:
 237         video_filename = cmdl_opts.outfile
 238
 239 # Check name
 240 if not video_filename.lower().endswith('.flv'):
 241         sys.stderr.write('Warning: video file name does not end in .flv\n')
 242
 243 # Test writable file
 244 if not cmdl_opts.simulate:
 245         try:
 246                 disk_test = open(video_filename, 'wb')
 247                 disk_test.close()
 248
 249         except (OSError, IOError):
 250                 sys.exit('Error: unable to open %s for writing.' % video_filename)
 251
 252 # Install cookie and proxy handlers
 253 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
 254 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
 255
 256 # Log in and confirm age if needed
 257 if account_username is not None:
 258         url = const_login_url_str % video_url_id
 259         post = const_login_post_str % (video_url_id, account_username, account_password)
 260         download_step(False, 'Logging in', 'unable to log in', url, post)
 261
 262         url = const_age_url_str % video_url_id
 263         post = const_age_post_str % video_url_id
 264         download_step(False, 'Confirming age', 'unable to confirm age', url, post)
 265
 266 # Retrieve video webpage
 267 video_webpage = download_step(True, 'Retrieving video webpage', 'unable to retrieve video webpage', video_url)
 268
 269 # Extract video title if needed
 270 if cmdl_opts.use_title:
 271         video_title = extract_step('Extracting video title', 'unable to extract video title', const_video_title_re, video_webpage)
 272
 273 # Extract needed video URL parameters
 274 video_url_params = extract_step('Extracting video URL parameters', 'unable to extract URL parameters', const_video_url_params_re, video_webpage)
 275 video_url_real = const_video_url_real_str % video_url_params
 276
 277 # Retrieve video data
 278 try:
 279         video_data = perform_request(video_url_real)
 280         cond_print('Video data found at %s\n' % video_data.geturl())
 281
 282         # Abort here if in simulate mode
 283         if cmdl_opts.simulate:
 284                 sys.exit()
 285
 286         video_file = open(video_filename, 'wb')
 287         try:
 288                 video_len_str = '%sk' % to_k(long(video_data.info()['Content-length']))
 289         except KeyError:
 290                 video_len_str = '(unknown)'
 291
 292         byte_counter = 0
 293         block_size = const_initial_block_size
 294         while True:
 295                 cond_print('\rRetrieving video data... %sk of %s ' % (to_k(byte_counter), video_len_str))
 296                 before = time.time()
 297                 video_block = video_data.read(block_size)
 298                 after = time.time()
 299                 dl_bytes = len(video_block)
 300                 if dl_bytes == 0:
 301                         break
 302                 byte_counter += dl_bytes
 303                 video_file.write(video_block)
 304                 block_size = new_block_size(before, after, dl_bytes)
 305
 306         video_file.close()
 307         cond_print('done.\n')
 308         cond_print('Video data saved to %s\n' % video_filename)
 309
 310 except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError):
 311         cond_print('failed.\n')
 312         error_advice_exit('unable to download video data')
 313
 314 except KeyboardInterrupt:
 315         sys.exit('\n')
 316
 317 # Rename video file if needed
 318 if cmdl_opts.use_title:
 319         try:
 320                 final_filename = '%s-%s.flv' % (title_string_norm(video_title), video_url_id)
 321                 os.rename(video_filename, final_filename)
 322                 cond_print('Video file renamed to %s\n' % final_filename)
 323
 324         except OSError:
 325                 sys.stderr.write('Warning: unable to rename file.\n')
 326
 327         except KeyboardInterrupt:
 328                 sys.exit('\n')
 329
 330 # Finish
 331 sys.exit()