]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube-dl
Imported Debian patch 2008.03.22-1
[youtubedl] / youtube-dl
1 #!/usr/bin/env python
2 #
3 # Copyright (c) 2006-2008 Ricardo Garcia Gonzalez
4 #
5 # Permission is hereby granted, free of charge, to any person obtaining a
6 # copy of this software and associated documentation files (the "Software"),
7 # to deal in the Software without restriction, including without limitation
8 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 # and/or sell copies of the Software, and to permit persons to whom the
10 # Software is furnished to do so, subject to the following conditions:
11 #
12 # The above copyright notice and this permission notice shall be included
13 # in all copies or substantial portions of the Software.
14 #
15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 # OTHER DEALINGS IN THE SOFTWARE.
22 #
23 # Except as contained in this notice, the name(s) of the above copyright
24 # holders shall not be used in advertising or otherwise to promote the
25 # sale, use or other dealings in this Software without prior written
26 # authorization.
27 #
28 import getpass
29 import httplib
30 import math
31 import netrc
32 import optparse
33 import os
34 import re
35 import socket
36 import string
37 import sys
38 import time
39 import urllib2
40
41 # Global constants
42 const_1k = 1024
43 const_initial_block_size = 10 * const_1k
44 const_epsilon = 0.0001
45 const_timeout = 120
46
47 const_video_url_str = 'http://www.youtube.com/watch?v=%s'
48 const_video_url_re = re.compile(r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:v/|(?:watch(?:\.php)?)?\?(?:.+&)?v=))?([0-9A-Za-z_-]+)(?(1)[&/].*)?$')
49 const_video_url_format_suffix = '&fmt=%s'
50 const_best_quality_format = 18
51 const_login_url_str = 'http://www.youtube.com/login?next=/watch%%3Fv%%3D%s'
52 const_login_post_str = 'current_form=loginForm&next=%%2Fwatch%%3Fv%%3D%s&username=%s&password=%s&action_login=Log+In'
53 const_age_url_str = 'http://www.youtube.com/verify_age?next_url=/watch%%3Fv%%3D%s'
54 const_age_post_str = 'next_url=%%2Fwatch%%3Fv%%3D%s&action_confirm=Confirm'
55 const_url_t_param_re = re.compile(r', "t": "([^"]+)"')
56 const_video_url_real_str = 'http://www.youtube.com/get_video?video_id=%s&t=%s'
57 const_video_title_re = re.compile(r'<title>YouTube - ([^<]*)</title>', re.M | re.I)
58
59 # Print error message, followed by standard advice information, and then exit
60 def error_advice_exit(error_text):
61 sys.stderr.write('Error: %s.\n' % error_text)
62 sys.stderr.write('Try again several times. It may be a temporary problem.\n')
63 sys.stderr.write('Other typical problems:\n\n')
64 sys.stderr.write('* Video no longer exists.\n')
65 sys.stderr.write('* Video requires age confirmation but you did not provide an account.\n')
66 sys.stderr.write('* You provided the account data, but it is not valid.\n')
67 sys.stderr.write('* The connection was cut suddenly for some reason.\n')
68 sys.stderr.write('* YouTube changed their system, and the program no longer works.\n')
69 sys.stderr.write('\nTry to confirm you are able to view the video using a web browser.\n')
70 sys.stderr.write('Use the same video URL and account information, if needed, with this program.\n')
71 sys.stderr.write('When using a proxy, make sure http_proxy has http://host:port format.\n')
72 sys.stderr.write('Try again several times and contact me if the problem persists.\n')
73 sys.exit('\n')
74
75 # Wrapper to create custom requests with typical headers
76 def request_create(url, extra_headers, post_data):
77 retval = urllib2.Request(url)
78 if post_data is not None:
79 retval.add_data(post_data)
80 # Try to mimic Firefox, at least a little bit
81 retval.add_header('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11')
82 retval.add_header('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')
83 retval.add_header('Accept', 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5')
84 retval.add_header('Accept-Language', 'en-us,en;q=0.5')
85 if extra_headers is not None:
86 for header in extra_headers:
87 retval.add_header(header[0], header[1])
88 return retval
89
90 # Perform a request, process headers and return response
91 def perform_request(url, headers=None, data=None):
92 request = request_create(url, headers, data)
93 response = urllib2.urlopen(request)
94 return response
95
96 # Conditional print
97 def cond_print(str):
98 global cmdl_opts
99 if not (cmdl_opts.quiet or cmdl_opts.get_url):
100 sys.stdout.write(str)
101 sys.stdout.flush()
102
103 # Title string normalization
104 def title_string_norm(title):
105 title = ''.join((x in string.ascii_letters or x in string.digits) and x or ' ' for x in title)
106 title = '_'.join(title.split())
107 title = title.lower()
108 return title
109
110 # Generic download step
111 def download_step(return_data_flag, step_title, step_error, url, post_data=None):
112 try:
113 cond_print('%s... ' % step_title)
114 data = perform_request(url, data=post_data).read()
115 cond_print('done.\n')
116 if return_data_flag:
117 return data
118 return None
119
120 except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError, socket.error):
121 cond_print('failed.\n')
122 error_advice_exit(step_error)
123
124 except KeyboardInterrupt:
125 sys.exit('\n')
126
127 # Generic extract step
128 def extract_step(step_title, step_error, regexp, data):
129 try:
130 cond_print('%s... ' % step_title)
131 match = regexp.search(data)
132
133 if match is None:
134 cond_print('failed.\n')
135 error_advice_exit(step_error)
136
137 extracted_data = match.group(1)
138 cond_print('done.\n')
139 return extracted_data
140
141 except KeyboardInterrupt:
142 sys.exit('\n')
143
144 # Calculate new block size based on previous block size
145 def new_block_size(before, after, bytes):
146 new_min = max(bytes / 2.0, 1.0)
147 new_max = max(bytes * 2.0, 1.0)
148 dif = after - before
149 if dif < const_epsilon:
150 return int(new_max)
151 rate = bytes / dif
152 if rate > new_max:
153 return int(new_max)
154 if rate < new_min:
155 return int(new_min)
156 return int(rate)
157
158 # Get optimum 1k exponent to represent a number of bytes
159 def optimum_k_exp(num_bytes):
160 global const_1k
161 if num_bytes == 0:
162 return 0
163 return long(math.log(num_bytes, const_1k))
164
165 # Get optimum representation of number of bytes
166 def format_bytes(num_bytes):
167 global const_1k
168 try:
169 exp = optimum_k_exp(num_bytes)
170 suffix = 'bkMGTPEZY'[exp]
171 if exp == 0:
172 return '%s%s' % (num_bytes, suffix)
173 converted = float(num_bytes) / float(const_1k**exp)
174 return '%.2f%s' % (converted, suffix)
175 except IndexError:
176 sys.exit('Error: internal error formatting number of bytes.')
177
178 # Calculate ETA and return it in string format as MM:SS
179 def calc_eta(start, now, total, current):
180 dif = now - start
181 if current == 0 or dif < const_epsilon:
182 return '--:--'
183 rate = float(current) / dif
184 eta = long((total - current) / rate)
185 (eta_mins, eta_secs) = divmod(eta, 60)
186 if eta_mins > 99:
187 return '--:--'
188 return '%02d:%02d' % (eta_mins, eta_secs)
189
190 # Calculate speed and return it in string format
191 def calc_speed(start, now, bytes):
192 dif = now - start
193 if bytes == 0 or dif < const_epsilon:
194 return 'N/A b'
195 return format_bytes(float(bytes) / dif)
196
197
198 # Title string minimal transformation
199 def title_string_touch(title):
200 return title.replace(os.sep, '%')
201
202 # Create the command line options parser and parse command line
203 cmdl_usage = 'usage: %prog [options] video_url'
204 cmdl_version = '2008.03.22'
205 cmdl_parser = optparse.OptionParser(usage=cmdl_usage, version=cmdl_version, conflict_handler='resolve')
206 cmdl_parser.add_option('-h', '--help', action='help', help='print this help text and exit')
207 cmdl_parser.add_option('-v', '--version', action='version', help='print program version and exit')
208 cmdl_parser.add_option('-u', '--username', dest='username', metavar='USERNAME', help='account username')
209 cmdl_parser.add_option('-p', '--password', dest='password', metavar='PASSWORD', help='account password')
210 cmdl_parser.add_option('-o', '--output', dest='outfile', metavar='FILE', help='output video file name')
211 cmdl_parser.add_option('-q', '--quiet', action='store_true', dest='quiet', help='activates quiet mode')
212 cmdl_parser.add_option('-s', '--simulate', action='store_true', dest='simulate', help='do not download video')
213 cmdl_parser.add_option('-t', '--title', action='store_true', dest='use_title', help='use title in file name')
214 cmdl_parser.add_option('-l', '--literal', action='store_true', dest='use_literal', help='use literal title in file name')
215 cmdl_parser.add_option('-n', '--netrc', action='store_true', dest='use_netrc', help='use .netrc authentication data')
216 cmdl_parser.add_option('-g', '--get-url', action='store_true', dest='get_url', help='print final video URL only')
217 cmdl_parser.add_option('-2', '--title-too', action='store_true', dest='get_title', help='used with -g, print title too')
218 cmdl_parser.add_option('-f', '--format', dest='video_format', metavar='FORMAT', help='append &fmt=FORMAT to the URL')
219 cmdl_parser.add_option('-b', '--best-quality', action='store_true', dest='best_quality', help='alias for -f 18')
220 (cmdl_opts, cmdl_args) = cmdl_parser.parse_args()
221
222 # Set socket timeout
223 socket.setdefaulttimeout(const_timeout)
224
225 # Get video URL
226 if len(cmdl_args) != 1:
227 cmdl_parser.print_help()
228 sys.exit('\n')
229 video_url_cmdl = cmdl_args[0]
230
231 # Verify video URL format and convert to "standard" format
232 video_url_mo = const_video_url_re.match(video_url_cmdl)
233 if video_url_mo is None:
234 sys.exit('Error: URL does not seem to be a youtube video URL. If it is, report a bug.')
235 video_url_id = video_url_mo.group(2)
236 video_url = const_video_url_str % video_url_id
237
238 video_format = None
239 if cmdl_opts.best_quality:
240 video_format = const_best_quality_format
241 video_extension = '.mp4'
242 else:
243 video_extension = '.flv'
244
245 if cmdl_opts.video_format is not None:
246 if video_format is not None and video_format != cmdl_opts.video_format:
247 sys.exit('Error: conflicting video formats specified\n')
248 video_format = cmdl_opts.video_format
249
250 if video_format is not None:
251 video_url = '%s%s' % (video_url, const_video_url_format_suffix % video_format)
252
253 # Check conflicting options
254 if cmdl_opts.outfile is not None and (cmdl_opts.simulate or cmdl_opts.get_url):
255 sys.stderr.write('Warning: video file name given but will not be used.\n')
256
257 if cmdl_opts.outfile is not None and (cmdl_opts.use_title or cmdl_opts.use_literal):
258 sys.exit('Error: using the video title conflicts with using a given file name.')
259
260 if cmdl_opts.use_title and cmdl_opts.use_literal:
261 sys.exit('Error: cannot use title and literal title at the same time.')
262
263 if cmdl_opts.quiet and cmdl_opts.get_url:
264 sys.exit('Error: cannot be quiet and print final URL at the same time.')
265
266 # Incorrect option formatting
267 if cmdl_opts.username is None and cmdl_opts.password is not None:
268 sys.exit('Error: password give but username is missing.')
269
270 if cmdl_opts.use_netrc and (cmdl_opts.username is not None or cmdl_opts.password is not None):
271 sys.exit('Error: cannot use netrc and username/password at the same time.')
272
273 if cmdl_opts.get_url is None and cmdl_opts.get_title is not None:
274 sys.exit('Error: getting title requires getting URL.')
275
276 # Get account information if any
277 account_username = None
278 account_password = None
279
280 if cmdl_opts.use_netrc:
281 try:
282 info = netrc.netrc().authenticators('youtube')
283 if info is None:
284 sys.exit('Error: no authenticators for machine youtube.')
285 account_username = info[0]
286 account_password = info[2]
287 except IOError:
288 sys.exit('Error: unable to read .netrc file.')
289 except netrc.NetrcParseError:
290 sys.exit('Error: unable to parse .netrc file.')
291 else:
292 account_username = cmdl_opts.username
293 if account_username is not None:
294 if cmdl_opts.password is None:
295 account_password = getpass.getpass('Type YouTube password and press return: ')
296 else:
297 account_password = cmdl_opts.password
298
299 # Get output file name
300 if cmdl_opts.outfile is None:
301 video_filename = '%s%s' % (video_url_id, video_extension)
302 else:
303 video_filename = cmdl_opts.outfile
304
305 # Install cookie and proxy handlers
306 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
307 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
308
309 # Log in and confirm age if needed
310 if account_username is not None:
311 url = const_login_url_str % video_url_id
312 post = const_login_post_str % (video_url_id, account_username, account_password)
313 download_step(False, 'Logging in', 'unable to log in', url, post)
314
315 url = const_age_url_str % video_url_id
316 post = const_age_post_str % video_url_id
317 download_step(False, 'Confirming age', 'unable to confirm age', url, post)
318
319 # Retrieve video webpage
320 video_webpage = download_step(True, 'Retrieving video webpage', 'unable to retrieve video webpage', video_url)
321
322 # Extract video title if needed
323 if cmdl_opts.use_title or cmdl_opts.use_literal or cmdl_opts.get_title:
324 video_title = extract_step('Extracting video title', 'unable to extract video title', const_video_title_re, video_webpage)
325
326 # Extract needed video URL parameters
327 video_url_t_param = extract_step('Extracting URL "t" parameter', 'unable to extract URL "t" parameter', const_url_t_param_re, video_webpage)
328 video_url_real = const_video_url_real_str % (video_url_id, video_url_t_param)
329 if video_format is not None:
330 video_url_real = '%s%s' % (video_url_real, const_video_url_format_suffix % video_format)
331
332 # Rebuild filename if needed
333 if cmdl_opts.use_title or cmdl_opts.use_literal:
334 if cmdl_opts.use_title:
335 prefix = title_string_norm(video_title)
336 else:
337 prefix = title_string_touch(video_title)
338 video_filename = '%s-%s%s' % (prefix, video_url_id, video_extension)
339
340 # Check name
341 if not video_filename.lower().endswith(video_extension):
342 sys.stderr.write('Warning: video file name does not end in %s\n' % video_extension)
343
344 # Retrieve video data
345 try:
346 cond_print('Requesting video file... ')
347 video_data = perform_request(video_url_real)
348 cond_print('done.\n')
349 cond_print('Video data found at %s\n' % video_data.geturl())
350
351 if cmdl_opts.get_title:
352 print video_title
353
354 if cmdl_opts.get_url:
355 print video_data.geturl()
356
357 if cmdl_opts.simulate or cmdl_opts.get_url:
358 sys.exit()
359
360 try:
361 video_file = open(video_filename, 'wb')
362 except (IOError, OSError):
363 sys.exit('Error: unable to open "%s" for writing.' % video_filename)
364 try:
365 video_len = long(video_data.info()['Content-length'])
366 video_len_str = format_bytes(video_len)
367 except KeyError:
368 video_len = None
369 video_len_str = 'N/A'
370
371 byte_counter = 0
372 block_size = const_initial_block_size
373 start_time = time.time()
374 while True:
375 if video_len is not None:
376 percent = float(byte_counter) / float(video_len) * 100.0
377 percent_str = '%.1f' % percent
378 eta_str = calc_eta(start_time, time.time(), video_len, byte_counter)
379 else:
380 percent_str = '---.-'
381 eta_str = '--:--'
382 counter = format_bytes(byte_counter)
383 speed_str = calc_speed(start_time, time.time(), byte_counter)
384 cond_print('\rRetrieving video data: %5s%% (%8s of %s) at %8s/s ETA %s ' % (percent_str, counter, video_len_str, speed_str, eta_str))
385
386 before = time.time()
387 video_block = video_data.read(block_size)
388 after = time.time()
389 dl_bytes = len(video_block)
390 if dl_bytes == 0:
391 break
392 byte_counter += dl_bytes
393 video_file.write(video_block)
394 block_size = new_block_size(before, after, dl_bytes)
395
396 if video_len is not None and byte_counter != video_len:
397 error_advice_exit('server did not send the expected amount of data')
398
399 video_file.close()
400 cond_print('done.\n')
401 cond_print('Video data saved to %s\n' % video_filename)
402
403 except (urllib2.URLError, ValueError, httplib.HTTPException, TypeError, socket.error):
404 cond_print('failed.\n')
405 error_advice_exit('unable to download video data')
406
407 except KeyboardInterrupt:
408 sys.exit('\n')
409
410 # Finish
411 sys.exit()