Raphaël G. Git Repositories - youtubedl/blob - youtube-dl

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2006 Ricardo Garcia Gonzalez
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining a
   6 # copy of this software and associated documentation files (the "Software"),
   7 # to deal in the Software without restriction, including without limitation
   8 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 # and/or sell copies of the Software, and to permit persons to whom the
  10 # Software is furnished to do so, subject to the following conditions:
  11 #
  12 # The above copyright notice and this permission notice shall be included
  13 # in all copies or substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21 # OTHER DEALINGS IN THE SOFTWARE.
  22 #
  23 # Except as contained in this notice, the name(s) of the above copyright
  24 # holders shall not be used in advertising or otherwise to promote the
  25 # sale, use or other dealings in this Software without prior written
  26 # authorization.
  27 #
  28 import getpass
  29 import httplib
  30 import netrc
  31 import optparse
  32 import os
  33 import re
  34 import socket
  35 import string
  36 import sys
  37 import time
  38 import urllib2
  39
  40 # Global constants
  41 const_video_url_str = 'http://www.youtube.com/watch?v=%s'
  42 const_video_url_re = re.compile(r'(?:http://)?(?:www\d*\.)?youtube\.com/(?:v/|(?:watch(?:\.php)?)?\?v=)([^&]+).*')
  43 const_login_url_str = 'http://www.youtube.com/login?next=/watch%%3Fv%%3D%s'
  44 const_login_post_str = 'current_form=loginForm&next=%%2Fwatch%%3Fv%%3D%s&username=%s&password=%s&action_login=Log+In'
  45 const_age_url_str = 'http://www.youtube.com/verify_age?next_url=/watch%%3Fv%%3D%s'
  46 const_age_post_str = 'next_url=%%2Fwatch%%3Fv%%3D%s&action_confirm=Confirm'
  47 const_video_url_params_re = re.compile(r'player2\.swf\?([^"]+)"', re.M)
  48 const_video_url_real_str = 'http://www.youtube.com/get_video?%s'
  49 const_video_title_re = re.compile(r'<title>YouTube - ([^<]*)</title>', re.M | re.I)
  50 const_1k = 1024
  51 const_initial_block_size = 10 * const_1k
  52
  53 # Print error message, followed by standard advice information, and then exit
  54 def error_advice_exit(error_text):
  55         sys.stderr.write('Error: %s.\n' % error_text)
  56         sys.stderr.write('Try again several times. It may be a temporary problem.\n')
  57         sys.stderr.write('Other typical problems:\n\n')
  58         sys.stderr.write('\tVideo no longer exists.\n')
  59         sys.stderr.write('\tVideo requires age confirmation but you did not provide an account.\n')
  60         sys.stderr.write('\tYou provided the account data, but it is not valid.\n')
  61         sys.stderr.write('\tThe connection was cut suddenly for some reason.\n')
  62         sys.stderr.write('\tYouTube changed their system, and the program no longer works.\n')
  63         sys.stderr.write('\nTry to confirm you are able to view the video using a web browser.\n')
  64         sys.stderr.write('Use the same video URL and account information, if needed, with this program.\n')
  65         sys.stderr.write('When using a proxy, make sure http_proxy has http://host:port format.\n')
  66         sys.stderr.write('Try again several times and contact me if the problem persists.\n')
  67         sys.exit('\n')
  68
  69 # Wrapper to create custom requests with typical headers
  70 def request_create(url, data=None):
  71         retval = urllib2.Request(url)
  72         if data is not None:
  73                 retval.add_data(data)
  74         # Try to mimic Firefox, at least a little bit
  75         retval.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1) Gecko/20061010 Firefox/2.0')
  76         retval.add_header('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')
  77         retval.add_header('Accept', 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5')
  78         retval.add_header('Accept-Language', 'en-us,en;q=0.5')
  79         return retval
  80
  81 # Perform a request, process headers and return response
  82 def perform_request(url, data=None):
  83         request = request_create(url, data)
  84         response = urllib2.urlopen(request)
  85         return response
  86
  87 # Convert bytes to KiB
  88 def to_k(bytes):
  89         global const_1k
  90         return bytes / const_1k
  91
  92 # Conditional print
  93 def cond_print(str):
  94         global cmdl_opts
  95         if not cmdl_opts.quiet:
  96                 sys.stdout.write(str)
  97                 sys.stdout.flush()
  98
  99 # Title string normalization
 100 def title_string_norm(title):
 101         title = ''.join((x in string.ascii_letters or x in string.digits) and x or ' ' for x in title)
 102         title = '_'.join(title.split())
 103         title = title.lower()
 104         return title
 105
 106 # Generic download step
 107 def download_step(return_data_flag, step_title, step_error, url, post_data=None):
 108         try:
 109                 cond_print('%s... ' % step_title)
 110                 data = perform_request(url, post_data).read()
 111                 cond_print('done.\n')
 112                 if return_data_flag:
 113                         return data
 114                 return None
 115
 116         except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError, socket.error):
 117                 cond_print('failed.\n')
 118                 error_advice_exit(step_error)
 119
 120         except KeyboardInterrupt:
 121                 sys.exit('\n')
 122
 123 # Generic extract step
 124 def extract_step(step_title, step_error, regexp, data):
 125         try:
 126                 cond_print('%s... ' % step_title)
 127                 match = regexp.search(data)
 128
 129                 if match is None:
 130                         cond_print('failed.\n')
 131                         error_advice_exit(step_error)
 132
 133                 extracted_data = match.group(1)
 134                 cond_print('done.\n')
 135                 return extracted_data
 136
 137         except KeyboardInterrupt:
 138                 sys.exit('\n')
 139
 140 # Calculate new block size based on previous block size
 141 def new_block_size(before, after, bytes):
 142         new_min = max(bytes / 2, 1)
 143         new_max = max(bytes * 2, 1)
 144         dif = after - before
 145         if dif < 0.0001:
 146                 return new_max
 147         rate = int(bytes / dif)
 148         if rate > new_max:
 149                 return new_max
 150         if rate < new_min:
 151                 return new_min
 152         return rate
 153
 154 # Create the command line options parser and parse command line
 155 cmdl_usage = 'usage: %prog [options] video_url'
 156 cmdl_version = '2007.02.18'
 157 cmdl_parser = optparse.OptionParser(usage=cmdl_usage, version=cmdl_version, conflict_handler='resolve')
 158 cmdl_parser.add_option('-h', '--help', action='help', help='print this help text and exit')
 159 cmdl_parser.add_option('-v', '--version', action='version', help='print program version and exit')
 160 cmdl_parser.add_option('-u', '--username', dest='username', metavar='USERNAME', help='account username')
 161 cmdl_parser.add_option('-p', '--password', dest='password', metavar='PASSWORD', help='account password')
 162 cmdl_parser.add_option('-o', '--output', dest='outfile', metavar='FILE', help='output video file name')
 163 cmdl_parser.add_option('-q', '--quiet', action='store_true', dest='quiet', help='activates quiet mode')
 164 cmdl_parser.add_option('-s', '--simulate', action='store_true', dest='simulate', help='do not download video')
 165 cmdl_parser.add_option('-t', '--title', action='store_true', dest='use_title', help='use title in file name')
 166 cmdl_parser.add_option('-n', '--netrc', action='store_true', dest='use_netrc', help='use .netrc authentication data')
 167 (cmdl_opts, cmdl_args) = cmdl_parser.parse_args()
 168
 169 # Get video URL
 170 if len(cmdl_args) != 1:
 171         cmdl_parser.print_help()
 172         sys.exit('\n')
 173 video_url_cmdl = cmdl_args[0]
 174
 175 # Verify video URL format and convert to "standard" format
 176 video_url_mo = const_video_url_re.match(video_url_cmdl)
 177 if video_url_mo is None:
 178         sys.exit('Error: URL does not seem to be a youtube video URL. If it is, report a bug.')
 179 video_url_id = video_url_mo.group(1)
 180 video_url = const_video_url_str % video_url_id
 181
 182 # Check conflicting options
 183 if cmdl_opts.outfile is not None and cmdl_opts.simulate:
 184         sys.stderr.write('Warning: video file name given but will not be used.\n')
 185
 186 if cmdl_opts.outfile is not None and cmdl_opts.use_title:
 187         sys.exit('Error: using the video title conflicts with using a given file name.')
 188
 189 if cmdl_opts.use_netrc and cmdl_opts.password is not None:
 190         sys.exit('Error: using netrc conflicts with giving command line password.')
 191
 192 # Incorrect option formatting
 193 if cmdl_opts.username is None and cmdl_opts.password is not None:
 194         sys.exit('Error: password give but username is missing.')
 195
 196 # Get account information if any
 197 account_username = None
 198 account_password = None
 199
 200 if cmdl_opts.use_netrc:
 201         try:
 202                 info = netrc.netrc().authenticators('youtube')
 203                 if info is None:
 204                         sys.exit('Error: no authenticators for machine youtube.')
 205                 netrc_username = info[0]
 206                 netrc_password = info[2]
 207         except IOError:
 208                 sys.exit('Error: unable to read .netrc file.')
 209         except netrc.NetrcParseError:
 210                 sys.exit('Error: unable to parse .netrc file.')
 211
 212 if cmdl_opts.password is not None:
 213         account_username = cmdl_opts.username
 214         account_password = cmdl_opts.password
 215 else:
 216         if cmdl_opts.username is not None and cmdl_opts.use_netrc:
 217                 if cmdl_opts.username != netrc_username:
 218                         sys.exit('Error: conflicting username from .netrc and command line options.')
 219                 account_username = cmdl_opts.username
 220                 account_password = netrc_password
 221         elif cmdl_opts.username is not None:
 222                 account_username = cmdl_opts.username
 223                 account_password = getpass.getpass('Type YouTube password and press return: ')
 224         elif cmdl_opts.use_netrc:
 225                 if len(netrc_username) == 0:
 226                         sys.exit('Error: empty username in .netrc file.')
 227                 account_username = netrc_username
 228                 account_password = netrc_password
 229
 230 # Get output file name
 231 if cmdl_opts.outfile is None:
 232         video_filename = '%s.flv' % video_url_id
 233 else:
 234         video_filename = cmdl_opts.outfile
 235
 236 # Check name
 237 if not video_filename.lower().endswith('.flv'):
 238         sys.stderr.write('Warning: video file name does not end in .flv\n')
 239
 240 # Test writable file
 241 if not cmdl_opts.simulate:
 242         try:
 243                 disk_test = open(video_filename, 'wb')
 244                 disk_test.close()
 245
 246         except (OSError, IOError):
 247                 sys.exit('Error: unable to open %s for writing.' % video_filename)
 248
 249 # Install cookie and proxy handlers
 250 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
 251 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
 252
 253 # Log in and confirm age if needed
 254 if account_username is not None:
 255         url = const_login_url_str % video_url_id
 256         post = const_login_post_str % (video_url_id, account_username, account_password)
 257         download_step(False, 'Logging in', 'unable to log in', url, post)
 258
 259         url = const_age_url_str % video_url_id
 260         post = const_age_post_str % video_url_id
 261         download_step(False, 'Confirming age', 'unable to confirm age', url, post)
 262
 263 # Retrieve video webpage
 264 video_webpage = download_step(True, 'Retrieving video webpage', 'unable to retrieve video webpage', video_url)
 265
 266 # Extract video title if needed
 267 if cmdl_opts.use_title:
 268         video_title = extract_step('Extracting video title', 'unable to extract video title', const_video_title_re, video_webpage)
 269
 270 # Extract needed video URL parameters
 271 video_url_params = extract_step('Extracting video URL parameters', 'unable to extract URL parameters', const_video_url_params_re, video_webpage)
 272 video_url_real = const_video_url_real_str % video_url_params
 273
 274 # Retrieve video data
 275 try:
 276         video_data = perform_request(video_url_real)
 277         cond_print('Video data found at %s\n' % video_data.geturl())
 278
 279         # Abort here if in simulate mode
 280         if cmdl_opts.simulate:
 281                 sys.exit()
 282
 283         video_file = open(video_filename, 'wb')
 284         try:
 285                 video_len_str = '%sk' % to_k(long(video_data.info()['Content-length']))
 286         except KeyError:
 287                 video_len_str = '(unknown)'
 288
 289         byte_counter = 0
 290         block_size = const_initial_block_size
 291         while True:
 292                 cond_print('\rRetrieving video data... %sk of %s ' % (to_k(byte_counter), video_len_str))
 293                 before = time.time()
 294                 video_block = video_data.read(block_size)
 295                 after = time.time()
 296                 dl_bytes = len(video_block)
 297                 if dl_bytes == 0:
 298                         break
 299                 byte_counter += dl_bytes
 300                 video_file.write(video_block)
 301                 block_size = new_block_size(before, after, dl_bytes)
 302
 303         video_file.close()
 304         cond_print('done.\n')
 305         cond_print('Video data saved to %s\n' % video_filename)
 306
 307 except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError, socket.error):
 308         cond_print('failed.\n')
 309         error_advice_exit('unable to download video data')
 310
 311 except KeyboardInterrupt:
 312         sys.exit('\n')
 313
 314 # Rename video file if needed
 315 if cmdl_opts.use_title:
 316         try:
 317                 final_filename = '%s-%s.flv' % (title_string_norm(video_title), video_url_id)
 318                 os.rename(video_filename, final_filename)
 319                 cond_print('Video file renamed to %s\n' % final_filename)
 320
 321         except OSError:
 322                 sys.stderr.write('Warning: unable to rename file.\n')
 323
 324         except KeyboardInterrupt:
 325                 sys.exit('\n')
 326
 327 # Finish
 328 sys.exit()