]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py
Merge tag 'upstream/2013.07.10'
[youtubedl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import absolute_import
5
6 import io
7 import os
8 import re
9 import shutil
10 import socket
11 import sys
12 import time
13 import traceback
14
15 from .utils import *
16 from .extractor import get_info_extractor, gen_extractors
17 from .FileDownloader import FileDownloader
18
19
20 class YoutubeDL(object):
21 """YoutubeDL class.
22
23 YoutubeDL objects are the ones responsible of downloading the
24 actual video file and writing it to disk if the user has requested
25 it, among some other tasks. In most cases there should be one per
26 program. As, given a video URL, the downloader doesn't know how to
27 extract all the needed information, task that InfoExtractors do, it
28 has to pass the URL to one of them.
29
30 For this, YoutubeDL objects have a method that allows
31 InfoExtractors to be registered in a given order. When it is passed
32 a URL, the YoutubeDL object handles it to the first InfoExtractor it
33 finds that reports being able to handle it. The InfoExtractor extracts
34 all the information about the video or videos the URL refers to, and
35 YoutubeDL process the extracted information, possibly using a File
36 Downloader to download the video.
37
38 YoutubeDL objects accept a lot of parameters. In order not to saturate
39 the object constructor with arguments, it receives a dictionary of
40 options instead. These options are available through the params
41 attribute for the InfoExtractors to use. The YoutubeDL also
42 registers itself as the downloader in charge for the InfoExtractors
43 that are added to it, so this is a "mutual registration".
44
45 Available options:
46
47 username: Username for authentication purposes.
48 password: Password for authentication purposes.
49 videopassword: Password for acces a video.
50 usenetrc: Use netrc for authentication instead.
51 verbose: Print additional info to stdout.
52 quiet: Do not print messages to stdout.
53 forceurl: Force printing final URL.
54 forcetitle: Force printing title.
55 forceid: Force printing ID.
56 forcethumbnail: Force printing thumbnail URL.
57 forcedescription: Force printing description.
58 forcefilename: Force printing final filename.
59 simulate: Do not download the video files.
60 format: Video format code.
61 format_limit: Highest quality format to try.
62 outtmpl: Template for output names.
63 restrictfilenames: Do not allow "&" and spaces in file names
64 ignoreerrors: Do not stop on download errors.
65 nooverwrites: Prevent overwriting files.
66 playliststart: Playlist item to start at.
67 playlistend: Playlist item to end at.
68 matchtitle: Download only matching titles.
69 rejecttitle: Reject downloads for matching titles.
70 logtostderr: Log messages to stderr instead of stdout.
71 writedescription: Write the video description to a .description file
72 writeinfojson: Write the video description to a .info.json file
73 writethumbnail: Write the thumbnail image to a file
74 writesubtitles: Write the video subtitles to a file
75 writeautomaticsub: Write the automatic subtitles to a file
76 allsubtitles: Downloads all the subtitles of the video
77 listsubtitles: Lists all available subtitles for the video
78 subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
79 subtitleslang: Language of the subtitles to download
80 keepvideo: Keep the video file after post-processing
81 daterange: A DateRange object, download only if the upload_date is in the range.
82 skip_download: Skip the actual download of the video file
83
84 The following parameters are not used by YoutubeDL itself, they are used by
85 the FileDownloader:
86 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
87 noresizebuffer, retries, continuedl, noprogress, consoletitle
88 """
89
90 params = None
91 _ies = []
92 _pps = []
93 _download_retcode = None
94 _num_downloads = None
95 _screen_file = None
96
97 def __init__(self, params):
98 """Create a FileDownloader object with the given options."""
99 self._ies = []
100 self._pps = []
101 self._progress_hooks = []
102 self._download_retcode = 0
103 self._num_downloads = 0
104 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
105 self.params = params
106 self.fd = FileDownloader(self, self.params)
107
108 if '%(stitle)s' in self.params['outtmpl']:
109 self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
110
111 def add_info_extractor(self, ie):
112 """Add an InfoExtractor object to the end of the list."""
113 self._ies.append(ie)
114 ie.set_downloader(self)
115
116 def add_default_info_extractors(self):
117 """
118 Add the InfoExtractors returned by gen_extractors to the end of the list
119 """
120 for ie in gen_extractors():
121 self.add_info_extractor(ie)
122
123 def add_post_processor(self, pp):
124 """Add a PostProcessor object to the end of the chain."""
125 self._pps.append(pp)
126 pp.set_downloader(self)
127
128 def to_screen(self, message, skip_eol=False):
129 """Print message to stdout if not in quiet mode."""
130 assert type(message) == type(u'')
131 if not self.params.get('quiet', False):
132 terminator = [u'\n', u''][skip_eol]
133 output = message + terminator
134 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
135 output = output.encode(preferredencoding(), 'ignore')
136 self._screen_file.write(output)
137 self._screen_file.flush()
138
139 def to_stderr(self, message):
140 """Print message to stderr."""
141 assert type(message) == type(u'')
142 output = message + u'\n'
143 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
144 output = output.encode(preferredencoding())
145 sys.stderr.write(output)
146
147 def fixed_template(self):
148 """Checks if the output template is fixed."""
149 return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
150
151 def trouble(self, message=None, tb=None):
152 """Determine action to take when a download problem appears.
153
154 Depending on if the downloader has been configured to ignore
155 download errors or not, this method may throw an exception or
156 not when errors are found, after printing the message.
157
158 tb, if given, is additional traceback information.
159 """
160 if message is not None:
161 self.to_stderr(message)
162 if self.params.get('verbose'):
163 if tb is None:
164 if sys.exc_info()[0]: # if .trouble has been called from an except block
165 tb = u''
166 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
167 tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
168 tb += compat_str(traceback.format_exc())
169 else:
170 tb_data = traceback.format_list(traceback.extract_stack())
171 tb = u''.join(tb_data)
172 self.to_stderr(tb)
173 if not self.params.get('ignoreerrors', False):
174 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
175 exc_info = sys.exc_info()[1].exc_info
176 else:
177 exc_info = sys.exc_info()
178 raise DownloadError(message, exc_info)
179 self._download_retcode = 1
180
181 def report_warning(self, message):
182 '''
183 Print the message to stderr, it will be prefixed with 'WARNING:'
184 If stderr is a tty file the 'WARNING:' will be colored
185 '''
186 if sys.stderr.isatty() and os.name != 'nt':
187 _msg_header=u'\033[0;33mWARNING:\033[0m'
188 else:
189 _msg_header=u'WARNING:'
190 warning_message=u'%s %s' % (_msg_header,message)
191 self.to_stderr(warning_message)
192
193 def report_error(self, message, tb=None):
194 '''
195 Do the same as trouble, but prefixes the message with 'ERROR:', colored
196 in red if stderr is a tty file.
197 '''
198 if sys.stderr.isatty() and os.name != 'nt':
199 _msg_header = u'\033[0;31mERROR:\033[0m'
200 else:
201 _msg_header = u'ERROR:'
202 error_message = u'%s %s' % (_msg_header, message)
203 self.trouble(error_message, tb)
204
205 def slow_down(self, start_time, byte_counter):
206 """Sleep if the download speed is over the rate limit."""
207 rate_limit = self.params.get('ratelimit', None)
208 if rate_limit is None or byte_counter == 0:
209 return
210 now = time.time()
211 elapsed = now - start_time
212 if elapsed <= 0.0:
213 return
214 speed = float(byte_counter) / elapsed
215 if speed > rate_limit:
216 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
217
218 def report_writedescription(self, descfn):
219 """ Report that the description file is being written """
220 self.to_screen(u'[info] Writing video description to: ' + descfn)
221
222 def report_writesubtitles(self, sub_filename):
223 """ Report that the subtitles file is being written """
224 self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
225
226 def report_writeinfojson(self, infofn):
227 """ Report that the metadata file has been written """
228 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
229
230 def report_file_already_downloaded(self, file_name):
231 """Report file has already been fully downloaded."""
232 try:
233 self.to_screen(u'[download] %s has already been downloaded' % file_name)
234 except (UnicodeEncodeError) as err:
235 self.to_screen(u'[download] The file has already been downloaded')
236
237 def increment_downloads(self):
238 """Increment the ordinal that assigns a number to each file."""
239 self._num_downloads += 1
240
241 def prepare_filename(self, info_dict):
242 """Generate the output filename."""
243 try:
244 template_dict = dict(info_dict)
245
246 template_dict['epoch'] = int(time.time())
247 autonumber_size = self.params.get('autonumber_size')
248 if autonumber_size is None:
249 autonumber_size = 5
250 autonumber_templ = u'%0' + str(autonumber_size) + u'd'
251 template_dict['autonumber'] = autonumber_templ % self._num_downloads
252 if template_dict['playlist_index'] is not None:
253 template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
254
255 sanitize = lambda k,v: sanitize_filename(
256 u'NA' if v is None else compat_str(v),
257 restricted=self.params.get('restrictfilenames'),
258 is_id=(k==u'id'))
259 template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
260
261 filename = self.params['outtmpl'] % template_dict
262 return filename
263 except KeyError as err:
264 self.report_error(u'Erroneous output template')
265 return None
266 except ValueError as err:
267 self.report_error(u'Insufficient system charset ' + repr(preferredencoding()))
268 return None
269
270 def _match_entry(self, info_dict):
271 """ Returns None iff the file should be downloaded """
272
273 title = info_dict['title']
274 matchtitle = self.params.get('matchtitle', False)
275 if matchtitle:
276 if not re.search(matchtitle, title, re.IGNORECASE):
277 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
278 rejecttitle = self.params.get('rejecttitle', False)
279 if rejecttitle:
280 if re.search(rejecttitle, title, re.IGNORECASE):
281 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
282 date = info_dict.get('upload_date', None)
283 if date is not None:
284 dateRange = self.params.get('daterange', DateRange())
285 if date not in dateRange:
286 return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
287 return None
288
289 def extract_info(self, url, download=True, ie_key=None, extra_info={}):
290 '''
291 Returns a list with a dictionary for each video we find.
292 If 'download', also downloads the videos.
293 extra_info is a dict containing the extra values to add to each result
294 '''
295
296 if ie_key:
297 ie = get_info_extractor(ie_key)()
298 ie.set_downloader(self)
299 ies = [ie]
300 else:
301 ies = self._ies
302
303 for ie in ies:
304 if not ie.suitable(url):
305 continue
306
307 if not ie.working():
308 self.report_warning(u'The program functionality for this site has been marked as broken, '
309 u'and will probably not work.')
310
311 try:
312 ie_result = ie.extract(url)
313 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
314 break
315 if isinstance(ie_result, list):
316 # Backwards compatibility: old IE result format
317 for result in ie_result:
318 result.update(extra_info)
319 ie_result = {
320 '_type': 'compat_list',
321 'entries': ie_result,
322 }
323 else:
324 ie_result.update(extra_info)
325 if 'extractor' not in ie_result:
326 ie_result['extractor'] = ie.IE_NAME
327 return self.process_ie_result(ie_result, download=download)
328 except ExtractorError as de: # An error we somewhat expected
329 self.report_error(compat_str(de), de.format_traceback())
330 break
331 except Exception as e:
332 if self.params.get('ignoreerrors', False):
333 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
334 break
335 else:
336 raise
337 else:
338 self.report_error(u'no suitable InfoExtractor: %s' % url)
339
340 def process_ie_result(self, ie_result, download=True, extra_info={}):
341 """
342 Take the result of the ie(may be modified) and resolve all unresolved
343 references (URLs, playlist items).
344
345 It will also download the videos if 'download'.
346 Returns the resolved ie_result.
347 """
348
349 result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
350 if result_type == 'video':
351 if 'playlist' not in ie_result:
352 # It isn't part of a playlist
353 ie_result['playlist'] = None
354 ie_result['playlist_index'] = None
355 if download:
356 self.process_info(ie_result)
357 return ie_result
358 elif result_type == 'url':
359 # We have to add extra_info to the results because it may be
360 # contained in a playlist
361 return self.extract_info(ie_result['url'],
362 download,
363 ie_key=ie_result.get('ie_key'),
364 extra_info=extra_info)
365 elif result_type == 'playlist':
366 # We process each entry in the playlist
367 playlist = ie_result.get('title', None) or ie_result.get('id', None)
368 self.to_screen(u'[download] Downloading playlist: %s' % playlist)
369
370 playlist_results = []
371
372 n_all_entries = len(ie_result['entries'])
373 playliststart = self.params.get('playliststart', 1) - 1
374 playlistend = self.params.get('playlistend', -1)
375
376 if playlistend == -1:
377 entries = ie_result['entries'][playliststart:]
378 else:
379 entries = ie_result['entries'][playliststart:playlistend]
380
381 n_entries = len(entries)
382
383 self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
384 (ie_result['extractor'], playlist, n_all_entries, n_entries))
385
386 for i,entry in enumerate(entries,1):
387 self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
388 extra = {
389 'playlist': playlist,
390 'playlist_index': i + playliststart,
391 }
392 if not 'extractor' in entry:
393 # We set the extractor, if it's an url it will be set then to
394 # the new extractor, but if it's already a video we must make
395 # sure it's present: see issue #877
396 entry['extractor'] = ie_result['extractor']
397 entry_result = self.process_ie_result(entry,
398 download=download,
399 extra_info=extra)
400 playlist_results.append(entry_result)
401 ie_result['entries'] = playlist_results
402 return ie_result
403 elif result_type == 'compat_list':
404 def _fixup(r):
405 r.setdefault('extractor', ie_result['extractor'])
406 return r
407 ie_result['entries'] = [
408 self.process_ie_result(_fixup(r), download=download)
409 for r in ie_result['entries']
410 ]
411 return ie_result
412 else:
413 raise Exception('Invalid result type: %s' % result_type)
414
415 def process_info(self, info_dict):
416 """Process a single resolved IE result."""
417
418 assert info_dict.get('_type', 'video') == 'video'
419 #We increment the download the download count here to match the previous behaviour.
420 self.increment_downloads()
421
422 info_dict['fulltitle'] = info_dict['title']
423 if len(info_dict['title']) > 200:
424 info_dict['title'] = info_dict['title'][:197] + u'...'
425
426 # Keep for backwards compatibility
427 info_dict['stitle'] = info_dict['title']
428
429 if not 'format' in info_dict:
430 info_dict['format'] = info_dict['ext']
431
432 reason = self._match_entry(info_dict)
433 if reason is not None:
434 self.to_screen(u'[download] ' + reason)
435 return
436
437 max_downloads = self.params.get('max_downloads')
438 if max_downloads is not None:
439 if self._num_downloads > int(max_downloads):
440 raise MaxDownloadsReached()
441
442 filename = self.prepare_filename(info_dict)
443
444 # Forced printings
445 if self.params.get('forcetitle', False):
446 compat_print(info_dict['title'])
447 if self.params.get('forceid', False):
448 compat_print(info_dict['id'])
449 if self.params.get('forceurl', False):
450 compat_print(info_dict['url'])
451 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
452 compat_print(info_dict['thumbnail'])
453 if self.params.get('forcedescription', False) and 'description' in info_dict:
454 compat_print(info_dict['description'])
455 if self.params.get('forcefilename', False) and filename is not None:
456 compat_print(filename)
457 if self.params.get('forceformat', False):
458 compat_print(info_dict['format'])
459
460 # Do nothing else if in simulate mode
461 if self.params.get('simulate', False):
462 return
463
464 if filename is None:
465 return
466
467 try:
468 dn = os.path.dirname(encodeFilename(filename))
469 if dn != '' and not os.path.exists(dn):
470 os.makedirs(dn)
471 except (OSError, IOError) as err:
472 self.report_error(u'unable to create directory ' + compat_str(err))
473 return
474
475 if self.params.get('writedescription', False):
476 try:
477 descfn = filename + u'.description'
478 self.report_writedescription(descfn)
479 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
480 descfile.write(info_dict['description'])
481 except (OSError, IOError):
482 self.report_error(u'Cannot write description file ' + descfn)
483 return
484
485 if (self.params.get('writesubtitles', False) or self.params.get('writeautomaticsub')) and 'subtitles' in info_dict and info_dict['subtitles']:
486 # subtitles download errors are already managed as troubles in relevant IE
487 # that way it will silently go on when used with unsupporting IE
488 subtitle = info_dict['subtitles'][0]
489 (sub_error, sub_lang, sub) = subtitle
490 sub_format = self.params.get('subtitlesformat')
491 if sub_error:
492 self.report_warning("Some error while getting the subtitles")
493 else:
494 try:
495 sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
496 self.report_writesubtitles(sub_filename)
497 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
498 subfile.write(sub)
499 except (OSError, IOError):
500 self.report_error(u'Cannot write subtitles file ' + descfn)
501 return
502
503 if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
504 subtitles = info_dict['subtitles']
505 sub_format = self.params.get('subtitlesformat')
506 for subtitle in subtitles:
507 (sub_error, sub_lang, sub) = subtitle
508 if sub_error:
509 self.report_warning("Some error while getting the subtitles")
510 else:
511 try:
512 sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
513 self.report_writesubtitles(sub_filename)
514 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
515 subfile.write(sub)
516 except (OSError, IOError):
517 self.report_error(u'Cannot write subtitles file ' + descfn)
518 return
519
520 if self.params.get('writeinfojson', False):
521 infofn = filename + u'.info.json'
522 self.report_writeinfojson(infofn)
523 try:
524 json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
525 write_json_file(json_info_dict, encodeFilename(infofn))
526 except (OSError, IOError):
527 self.report_error(u'Cannot write metadata to JSON file ' + infofn)
528 return
529
530 if self.params.get('writethumbnail', False):
531 if 'thumbnail' in info_dict:
532 thumb_format = info_dict['thumbnail'].rpartition(u'/')[2].rpartition(u'.')[2]
533 if not thumb_format:
534 thumb_format = 'jpg'
535 thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
536 self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
537 (info_dict['extractor'], info_dict['id']))
538 uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
539 with open(thumb_filename, 'wb') as thumbf:
540 shutil.copyfileobj(uf, thumbf)
541 self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
542 (info_dict['extractor'], info_dict['id'], thumb_filename))
543
544 if not self.params.get('skip_download', False):
545 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
546 success = True
547 else:
548 try:
549 success = self.fd._do_download(filename, info_dict)
550 except (OSError, IOError) as err:
551 raise UnavailableVideoError()
552 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
553 self.report_error(u'unable to download video data: %s' % str(err))
554 return
555 except (ContentTooShortError, ) as err:
556 self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
557 return
558
559 if success:
560 try:
561 self.post_process(filename, info_dict)
562 except (PostProcessingError) as err:
563 self.report_error(u'postprocessing: %s' % str(err))
564 return
565
566 def download(self, url_list):
567 """Download a given list of URLs."""
568 if len(url_list) > 1 and self.fixed_template():
569 raise SameFileError(self.params['outtmpl'])
570
571 for url in url_list:
572 try:
573 #It also downloads the videos
574 videos = self.extract_info(url)
575 except UnavailableVideoError:
576 self.report_error(u'unable to download video')
577 except MaxDownloadsReached:
578 self.to_screen(u'[info] Maximum number of downloaded files reached.')
579 raise
580
581 return self._download_retcode
582
583 def post_process(self, filename, ie_info):
584 """Run all the postprocessors on the given file."""
585 info = dict(ie_info)
586 info['filepath'] = filename
587 keep_video = None
588 for pp in self._pps:
589 try:
590 keep_video_wish,new_info = pp.run(info)
591 if keep_video_wish is not None:
592 if keep_video_wish:
593 keep_video = keep_video_wish
594 elif keep_video is None:
595 # No clear decision yet, let IE decide
596 keep_video = keep_video_wish
597 except PostProcessingError as e:
598 self.to_stderr(u'ERROR: ' + e.msg)
599 if keep_video is False and not self.params.get('keepvideo', False):
600 try:
601 self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
602 os.remove(encodeFilename(filename))
603 except (IOError, OSError):
604 self.report_warning(u'Unable to remove downloaded video file')