]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube-dl
Imported Debian patch 2007.03.27-1
[youtubedl] / youtube-dl
1 #!/usr/bin/env python
2 #
3 # Copyright (c) 2006 Ricardo Garcia Gonzalez
4 #
5 # Permission is hereby granted, free of charge, to any person obtaining a
6 # copy of this software and associated documentation files (the "Software"),
7 # to deal in the Software without restriction, including without limitation
8 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 # and/or sell copies of the Software, and to permit persons to whom the
10 # Software is furnished to do so, subject to the following conditions:
11 #
12 # The above copyright notice and this permission notice shall be included
13 # in all copies or substantial portions of the Software.
14 #
15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 # OTHER DEALINGS IN THE SOFTWARE.
22 #
23 # Except as contained in this notice, the name(s) of the above copyright
24 # holders shall not be used in advertising or otherwise to promote the
25 # sale, use or other dealings in this Software without prior written
26 # authorization.
27 #
28 import getpass
29 import httplib
30 import math
31 import netrc
32 import optparse
33 import os
34 import re
35 import socket
36 import string
37 import sys
38 import time
39 import urllib2
40
41 # Global constants
42 const_video_url_str = 'http://www.youtube.com/watch?v=%s'
43 const_video_url_re = re.compile(r'(?:http://)?(?:www\d*\.)?youtube\.com/(?:v/|(?:watch(?:\.php)?)?\?(?:.+&)?v=)([^&]+).*')
44 const_login_url_str = 'http://www.youtube.com/login?next=/watch%%3Fv%%3D%s'
45 const_login_post_str = 'current_form=loginForm&next=%%2Fwatch%%3Fv%%3D%s&username=%s&password=%s&action_login=Log+In'
46 const_age_url_str = 'http://www.youtube.com/verify_age?next_url=/watch%%3Fv%%3D%s'
47 const_age_post_str = 'next_url=%%2Fwatch%%3Fv%%3D%s&action_confirm=Confirm'
48 const_video_url_params_re = re.compile(r'player2\.swf\?([^"]+)"', re.M)
49 const_video_url_real_str = 'http://www.youtube.com/get_video?%s'
50 const_video_title_re = re.compile(r'<title>YouTube - ([^<]*)</title>', re.M | re.I)
51 const_1k = 1024
52 const_initial_block_size = 10 * const_1k
53
54 # Print error message, followed by standard advice information, and then exit
55 def error_advice_exit(error_text):
56 sys.stderr.write('Error: %s.\n' % error_text)
57 sys.stderr.write('Try again several times. It may be a temporary problem.\n')
58 sys.stderr.write('Other typical problems:\n\n')
59 sys.stderr.write('* Video no longer exists.\n')
60 sys.stderr.write('* Video requires age confirmation but you did not provide an account.\n')
61 sys.stderr.write('* You provided the account data, but it is not valid.\n')
62 sys.stderr.write('* The connection was cut suddenly for some reason.\n')
63 sys.stderr.write('* YouTube changed their system, and the program no longer works.\n')
64 sys.stderr.write('\nTry to confirm you are able to view the video using a web browser.\n')
65 sys.stderr.write('Use the same video URL and account information, if needed, with this program.\n')
66 sys.stderr.write('When using a proxy, make sure http_proxy has http://host:port format.\n')
67 sys.stderr.write('Try again several times and contact me if the problem persists.\n')
68 sys.exit('\n')
69
70 # Wrapper to create custom requests with typical headers
71 def request_create(url, data=None):
72 retval = urllib2.Request(url)
73 if data is not None:
74 retval.add_data(data)
75 # Try to mimic Firefox, at least a little bit
76 retval.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1) Gecko/20061010 Firefox/2.0')
77 retval.add_header('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')
78 retval.add_header('Accept', 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5')
79 retval.add_header('Accept-Language', 'en-us,en;q=0.5')
80 return retval
81
82 # Perform a request, process headers and return response
83 def perform_request(url, data=None):
84 request = request_create(url, data)
85 response = urllib2.urlopen(request)
86 return response
87
88 # Conditional print
89 def cond_print(str):
90 global cmdl_opts
91 if not (cmdl_opts.quiet or cmdl_opts.get_url):
92 sys.stdout.write(str)
93 sys.stdout.flush()
94
95 # Title string normalization
96 def title_string_norm(title):
97 title = ''.join((x in string.ascii_letters or x in string.digits) and x or ' ' for x in title)
98 title = '_'.join(title.split())
99 title = title.lower()
100 return title
101
102 # Title string minimal transformation
103 def title_string_touch(title):
104 return title.replace(os.sep, '%')
105
106 # Generic download step
107 def download_step(return_data_flag, step_title, step_error, url, post_data=None):
108 try:
109 cond_print('%s... ' % step_title)
110 data = perform_request(url, post_data).read()
111 cond_print('done.\n')
112 if return_data_flag:
113 return data
114 return None
115
116 except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError, socket.error):
117 cond_print('failed.\n')
118 error_advice_exit(step_error)
119
120 except KeyboardInterrupt:
121 sys.exit('\n')
122
123 # Generic extract step
124 def extract_step(step_title, step_error, regexp, data):
125 try:
126 cond_print('%s... ' % step_title)
127 match = regexp.search(data)
128
129 if match is None:
130 cond_print('failed.\n')
131 error_advice_exit(step_error)
132
133 extracted_data = match.group(1)
134 cond_print('done.\n')
135 return extracted_data
136
137 except KeyboardInterrupt:
138 sys.exit('\n')
139
140 # Calculate new block size based on previous block size
141 def new_block_size(before, after, bytes):
142 new_min = max(bytes / 2.0, 1.0)
143 new_max = max(bytes * 2.0, 1.0)
144 dif = after - before
145 if dif < 0.0001:
146 return int(new_max)
147 rate = bytes / dif
148 if rate > new_max:
149 return int(new_max)
150 if rate < new_min:
151 return int(new_min)
152 return int(rate)
153
154 # Get optimum 1k exponent to represent a number of bytes
155 def optimum_k_exp(num_bytes):
156 global const_1k
157 if num_bytes == 0:
158 return 0
159 return long(math.log(num_bytes, const_1k))
160
161 # Get optimum representation of number of bytes
162 def format_bytes(num_bytes):
163 global const_1k
164 try:
165 exp = optimum_k_exp(num_bytes)
166 suffix = 'bkMGTPEZY'[exp]
167 if exp == 0:
168 return '%s%s' % (num_bytes, suffix)
169 converted = float(num_bytes) / float(const_1k**exp)
170 return '%.2f%s' % (converted, suffix)
171 except IndexError:
172 sys.exit('Error: internal error formatting number of bytes.')
173
174 # Calculate ETA and return it in string format as MM:SS
175 def calc_eta(start, now, total, current):
176 if current == 0:
177 return '--:--'
178 rate = float(current) / (now - start)
179 eta = long((total - current) / rate)
180 eta_mins = eta / 60
181 eta_secs = eta % 60
182 if eta_mins > 99:
183 return '--:--'
184 return '%02d:%02d' % (eta_mins, eta_secs)
185
186 # Calculate speed and return it in string format
187 def calc_speed(start, now, bytes):
188 if bytes == 0:
189 return 'N/A b'
190 return format_bytes(float(bytes) / (now - start))
191
192 # Create the command line options parser and parse command line
193 cmdl_usage = 'usage: %prog [options] video_url'
194 cmdl_version = '2007.03.27'
195 cmdl_parser = optparse.OptionParser(usage=cmdl_usage, version=cmdl_version, conflict_handler='resolve')
196 cmdl_parser.add_option('-h', '--help', action='help', help='print this help text and exit')
197 cmdl_parser.add_option('-v', '--version', action='version', help='print program version and exit')
198 cmdl_parser.add_option('-u', '--username', dest='username', metavar='USERNAME', help='account username')
199 cmdl_parser.add_option('-p', '--password', dest='password', metavar='PASSWORD', help='account password')
200 cmdl_parser.add_option('-o', '--output', dest='outfile', metavar='FILE', help='output video file name')
201 cmdl_parser.add_option('-q', '--quiet', action='store_true', dest='quiet', help='activates quiet mode')
202 cmdl_parser.add_option('-s', '--simulate', action='store_true', dest='simulate', help='do not download video')
203 cmdl_parser.add_option('-t', '--title', action='store_true', dest='use_title', help='use title in file name')
204 cmdl_parser.add_option('-l', '--literal', action='store_true', dest='use_literal', help='use literal title in file name')
205 cmdl_parser.add_option('-n', '--netrc', action='store_true', dest='use_netrc', help='use .netrc authentication data')
206 cmdl_parser.add_option('-g', '--get-url', action='store_true', dest='get_url', help='print final video URL only')
207 (cmdl_opts, cmdl_args) = cmdl_parser.parse_args()
208
209 # Get video URL
210 if len(cmdl_args) != 1:
211 cmdl_parser.print_help()
212 sys.exit('\n')
213 video_url_cmdl = cmdl_args[0]
214
215 # Verify video URL format and convert to "standard" format
216 video_url_mo = const_video_url_re.match(video_url_cmdl)
217 if video_url_mo is None:
218 sys.exit('Error: URL does not seem to be a youtube video URL. If it is, report a bug.')
219 video_url_id = video_url_mo.group(1)
220 video_url = const_video_url_str % video_url_id
221
222 # Check conflicting options
223 if cmdl_opts.outfile is not None and (cmdl_opts.simulate or cmdl_opts.get_url):
224 sys.stderr.write('Warning: video file name given but will not be used.\n')
225
226 if cmdl_opts.outfile is not None and (cmdl_opts.use_title or cmdl_opts.use_literal):
227 sys.exit('Error: using the video title conflicts with using a given file name.')
228
229 if cmdl_opts.use_netrc and cmdl_opts.password is not None:
230 sys.exit('Error: using netrc conflicts with giving command line password.')
231
232 if cmdl_opts.use_title and cmdl_opts.use_literal:
233 sys.exit('Error: cannot use title and literal title at the same time.')
234
235 if cmdl_opts.quiet and cmdl_opts.get_url:
236 sys.exit('Error: cannot be quiet and print final URL at the same time.')
237
238 # Incorrect option formatting
239 if cmdl_opts.username is None and cmdl_opts.password is not None:
240 sys.exit('Error: password give but username is missing.')
241
242 # Get account information if any
243 account_username = None
244 account_password = None
245
246 if cmdl_opts.use_netrc:
247 try:
248 info = netrc.netrc().authenticators('youtube')
249 if info is None:
250 sys.exit('Error: no authenticators for machine youtube.')
251 netrc_username = info[0]
252 netrc_password = info[2]
253 except IOError:
254 sys.exit('Error: unable to read .netrc file.')
255 except netrc.NetrcParseError:
256 sys.exit('Error: unable to parse .netrc file.')
257
258 if cmdl_opts.password is not None:
259 account_username = cmdl_opts.username
260 account_password = cmdl_opts.password
261 else:
262 if cmdl_opts.username is not None and cmdl_opts.use_netrc:
263 if cmdl_opts.username != netrc_username:
264 sys.exit('Error: conflicting username from .netrc and command line options.')
265 account_username = cmdl_opts.username
266 account_password = netrc_password
267 elif cmdl_opts.username is not None:
268 account_username = cmdl_opts.username
269 account_password = getpass.getpass('Type YouTube password and press return: ')
270 elif cmdl_opts.use_netrc:
271 if len(netrc_username) == 0:
272 sys.exit('Error: empty username in .netrc file.')
273 account_username = netrc_username
274 account_password = netrc_password
275
276 # Get output file name
277 if cmdl_opts.outfile is None:
278 video_filename = '%s.flv' % video_url_id
279 else:
280 video_filename = cmdl_opts.outfile
281
282 # Check name
283 if not video_filename.lower().endswith('.flv'):
284 sys.stderr.write('Warning: video file name does not end in .flv\n')
285
286 # Test writable file
287 if not (cmdl_opts.simulate or cmdl_opts.get_url):
288 try:
289 disk_test = open(video_filename, 'wb')
290 disk_test.close()
291
292 except (OSError, IOError):
293 sys.exit('Error: unable to open %s for writing.' % video_filename)
294
295 # Install cookie and proxy handlers
296 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
297 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
298
299 # Log in and confirm age if needed
300 if account_username is not None:
301 url = const_login_url_str % video_url_id
302 post = const_login_post_str % (video_url_id, account_username, account_password)
303 download_step(False, 'Logging in', 'unable to log in', url, post)
304
305 url = const_age_url_str % video_url_id
306 post = const_age_post_str % video_url_id
307 download_step(False, 'Confirming age', 'unable to confirm age', url, post)
308
309 # Retrieve video webpage
310 video_webpage = download_step(True, 'Retrieving video webpage', 'unable to retrieve video webpage', video_url)
311
312 # Extract video title if needed
313 if cmdl_opts.use_title or cmdl_opts.use_literal:
314 video_title = extract_step('Extracting video title', 'unable to extract video title', const_video_title_re, video_webpage)
315
316 # Extract needed video URL parameters
317 video_url_params = extract_step('Extracting video URL parameters', 'unable to extract URL parameters', const_video_url_params_re, video_webpage)
318 video_url_real = const_video_url_real_str % video_url_params
319
320 # Retrieve video data
321 try:
322 video_data = perform_request(video_url_real)
323 cond_print('Video data found at %s\n' % video_data.geturl())
324
325 if cmdl_opts.get_url:
326 print video_data.geturl()
327
328 if cmdl_opts.simulate or cmdl_opts.get_url:
329 sys.exit()
330
331 video_file = open(video_filename, 'wb')
332 try:
333 video_len = long(video_data.info()['Content-length'])
334 video_len_str = format_bytes(video_len)
335 except KeyError:
336 video_len = None
337 video_len_str = 'N/A'
338
339 byte_counter = 0
340 block_size = const_initial_block_size
341 start_time = time.time()
342 while True:
343 if video_len is not None:
344 percent = float(byte_counter) / float(video_len) * 100.0
345 percent_str = '%.1f' % percent
346 eta_str = calc_eta(start_time, time.time(), video_len, byte_counter)
347 else:
348 percent_str = '---.-'
349 eta_str = '--:--'
350 counter = format_bytes(byte_counter)
351 speed_str = calc_speed(start_time, time.time(), byte_counter)
352 cond_print('\rRetrieving video data: %5s%% (%8s of %s) at %8s/s ETA %s ' % (percent_str, counter, video_len_str, speed_str, eta_str))
353
354 before = time.time()
355 video_block = video_data.read(block_size)
356 after = time.time()
357 dl_bytes = len(video_block)
358 if dl_bytes == 0:
359 break
360 byte_counter += dl_bytes
361 video_file.write(video_block)
362 block_size = new_block_size(before, after, dl_bytes)
363
364 if video_len is not None and byte_counter != video_len:
365 error_advice_exit('server did not send the expected amount of data')
366
367 video_file.close()
368 cond_print('done.\n')
369 cond_print('Video data saved to %s\n' % video_filename)
370
371 except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError, socket.error):
372 cond_print('failed.\n')
373 error_advice_exit('unable to download video data')
374
375 except KeyboardInterrupt:
376 sys.exit('\n')
377
378 # Rename video file if needed
379 if cmdl_opts.use_title or cmdl_opts.use_literal:
380 try:
381 if cmdl_opts.use_title:
382 prefix = title_string_norm(video_title)
383 else:
384 prefix = title_string_touch(video_title)
385 final_filename = '%s-%s.flv' % (prefix, video_url_id)
386 os.rename(video_filename, final_filename)
387 cond_print('Video file renamed to %s\n' % final_filename)
388
389 except OSError:
390 sys.stderr.write('Warning: unable to rename file.\n')
391
392 except KeyboardInterrupt:
393 sys.exit('\n')
394
395 # Finish
396 sys.exit()