]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/utils.py
Update changelog.
[youtubedl] / youtube_dl / utils.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 import errno
5 import gzip
6 import io
7 import json
8 import locale
9 import os
10 import re
11 import sys
12 import traceback
13 import zlib
14 import email.utils
15 import json
16 import datetime
17
18 try:
19 import urllib.request as compat_urllib_request
20 except ImportError: # Python 2
21 import urllib2 as compat_urllib_request
22
23 try:
24 import urllib.error as compat_urllib_error
25 except ImportError: # Python 2
26 import urllib2 as compat_urllib_error
27
28 try:
29 import urllib.parse as compat_urllib_parse
30 except ImportError: # Python 2
31 import urllib as compat_urllib_parse
32
33 try:
34 from urllib.parse import urlparse as compat_urllib_parse_urlparse
35 except ImportError: # Python 2
36 from urlparse import urlparse as compat_urllib_parse_urlparse
37
38 try:
39 import http.cookiejar as compat_cookiejar
40 except ImportError: # Python 2
41 import cookielib as compat_cookiejar
42
43 try:
44 import html.entities as compat_html_entities
45 except ImportError: # Python 2
46 import htmlentitydefs as compat_html_entities
47
48 try:
49 import html.parser as compat_html_parser
50 except ImportError: # Python 2
51 import HTMLParser as compat_html_parser
52
53 try:
54 import http.client as compat_http_client
55 except ImportError: # Python 2
56 import httplib as compat_http_client
57
58 try:
59 from subprocess import DEVNULL
60 compat_subprocess_get_DEVNULL = lambda: DEVNULL
61 except ImportError:
62 compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
63
64 try:
65 from urllib.parse import parse_qs as compat_parse_qs
66 except ImportError: # Python 2
67 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
68 # Python 2's version is apparently totally broken
69 def _unquote(string, encoding='utf-8', errors='replace'):
70 if string == '':
71 return string
72 res = string.split('%')
73 if len(res) == 1:
74 return string
75 if encoding is None:
76 encoding = 'utf-8'
77 if errors is None:
78 errors = 'replace'
79 # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
80 pct_sequence = b''
81 string = res[0]
82 for item in res[1:]:
83 try:
84 if not item:
85 raise ValueError
86 pct_sequence += item[:2].decode('hex')
87 rest = item[2:]
88 if not rest:
89 # This segment was just a single percent-encoded character.
90 # May be part of a sequence of code units, so delay decoding.
91 # (Stored in pct_sequence).
92 continue
93 except ValueError:
94 rest = '%' + item
95 # Encountered non-percent-encoded characters. Flush the current
96 # pct_sequence.
97 string += pct_sequence.decode(encoding, errors) + rest
98 pct_sequence = b''
99 if pct_sequence:
100 # Flush the final pct_sequence
101 string += pct_sequence.decode(encoding, errors)
102 return string
103
104 def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
105 encoding='utf-8', errors='replace'):
106 qs, _coerce_result = qs, unicode
107 pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
108 r = []
109 for name_value in pairs:
110 if not name_value and not strict_parsing:
111 continue
112 nv = name_value.split('=', 1)
113 if len(nv) != 2:
114 if strict_parsing:
115 raise ValueError("bad query field: %r" % (name_value,))
116 # Handle case of a control-name with no equal sign
117 if keep_blank_values:
118 nv.append('')
119 else:
120 continue
121 if len(nv[1]) or keep_blank_values:
122 name = nv[0].replace('+', ' ')
123 name = _unquote(name, encoding=encoding, errors=errors)
124 name = _coerce_result(name)
125 value = nv[1].replace('+', ' ')
126 value = _unquote(value, encoding=encoding, errors=errors)
127 value = _coerce_result(value)
128 r.append((name, value))
129 return r
130
131 def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
132 encoding='utf-8', errors='replace'):
133 parsed_result = {}
134 pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
135 encoding=encoding, errors=errors)
136 for name, value in pairs:
137 if name in parsed_result:
138 parsed_result[name].append(value)
139 else:
140 parsed_result[name] = [value]
141 return parsed_result
142
143 try:
144 compat_str = unicode # Python 2
145 except NameError:
146 compat_str = str
147
148 try:
149 compat_chr = unichr # Python 2
150 except NameError:
151 compat_chr = chr
152
153 std_headers = {
154 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
155 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
156 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
157 'Accept-Encoding': 'gzip, deflate',
158 'Accept-Language': 'en-us,en;q=0.5',
159 }
160
161 def preferredencoding():
162 """Get preferred encoding.
163
164 Returns the best encoding scheme for the system, based on
165 locale.getpreferredencoding() and some further tweaks.
166 """
167 try:
168 pref = locale.getpreferredencoding()
169 u'TEST'.encode(pref)
170 except:
171 pref = 'UTF-8'
172
173 return pref
174
175 if sys.version_info < (3,0):
176 def compat_print(s):
177 print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
178 else:
179 def compat_print(s):
180 assert type(s) == type(u'')
181 print(s)
182
183 # In Python 2.x, json.dump expects a bytestream.
184 # In Python 3.x, it writes to a character stream
185 if sys.version_info < (3,0):
186 def write_json_file(obj, fn):
187 with open(fn, 'wb') as f:
188 json.dump(obj, f)
189 else:
190 def write_json_file(obj, fn):
191 with open(fn, 'w', encoding='utf-8') as f:
192 json.dump(obj, f)
193
194 def htmlentity_transform(matchobj):
195 """Transforms an HTML entity to a character.
196
197 This function receives a match object and is intended to be used with
198 the re.sub() function.
199 """
200 entity = matchobj.group(1)
201
202 # Known non-numeric HTML entity
203 if entity in compat_html_entities.name2codepoint:
204 return compat_chr(compat_html_entities.name2codepoint[entity])
205
206 mobj = re.match(u'(?u)#(x?\\d+)', entity)
207 if mobj is not None:
208 numstr = mobj.group(1)
209 if numstr.startswith(u'x'):
210 base = 16
211 numstr = u'0%s' % numstr
212 else:
213 base = 10
214 return compat_chr(int(numstr, base))
215
216 # Unknown entity in name, return its literal representation
217 return (u'&%s;' % entity)
218
219 compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
220 class AttrParser(compat_html_parser.HTMLParser):
221 """Modified HTMLParser that isolates a tag with the specified attribute"""
222 def __init__(self, attribute, value):
223 self.attribute = attribute
224 self.value = value
225 self.result = None
226 self.started = False
227 self.depth = {}
228 self.html = None
229 self.watch_startpos = False
230 self.error_count = 0
231 compat_html_parser.HTMLParser.__init__(self)
232
233 def error(self, message):
234 if self.error_count > 10 or self.started:
235 raise compat_html_parser.HTMLParseError(message, self.getpos())
236 self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
237 self.error_count += 1
238 self.goahead(1)
239
240 def loads(self, html):
241 self.html = html
242 self.feed(html)
243 self.close()
244
245 def handle_starttag(self, tag, attrs):
246 attrs = dict(attrs)
247 if self.started:
248 self.find_startpos(None)
249 if self.attribute in attrs and attrs[self.attribute] == self.value:
250 self.result = [tag]
251 self.started = True
252 self.watch_startpos = True
253 if self.started:
254 if not tag in self.depth: self.depth[tag] = 0
255 self.depth[tag] += 1
256
257 def handle_endtag(self, tag):
258 if self.started:
259 if tag in self.depth: self.depth[tag] -= 1
260 if self.depth[self.result[0]] == 0:
261 self.started = False
262 self.result.append(self.getpos())
263
264 def find_startpos(self, x):
265 """Needed to put the start position of the result (self.result[1])
266 after the opening tag with the requested id"""
267 if self.watch_startpos:
268 self.watch_startpos = False
269 self.result.append(self.getpos())
270 handle_entityref = handle_charref = handle_data = handle_comment = \
271 handle_decl = handle_pi = unknown_decl = find_startpos
272
273 def get_result(self):
274 if self.result is None:
275 return None
276 if len(self.result) != 3:
277 return None
278 lines = self.html.split('\n')
279 lines = lines[self.result[1][0]-1:self.result[2][0]]
280 lines[0] = lines[0][self.result[1][1]:]
281 if len(lines) == 1:
282 lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
283 lines[-1] = lines[-1][:self.result[2][1]]
284 return '\n'.join(lines).strip()
285 # Hack for https://github.com/rg3/youtube-dl/issues/662
286 if sys.version_info < (2, 7, 3):
287 AttrParser.parse_endtag = (lambda self, i:
288 i + len("</scr'+'ipt>")
289 if self.rawdata[i:].startswith("</scr'+'ipt>")
290 else compat_html_parser.HTMLParser.parse_endtag(self, i))
291
292 def get_element_by_id(id, html):
293 """Return the content of the tag with the specified ID in the passed HTML document"""
294 return get_element_by_attribute("id", id, html)
295
296 def get_element_by_attribute(attribute, value, html):
297 """Return the content of the tag with the specified attribute in the passed HTML document"""
298 parser = AttrParser(attribute, value)
299 try:
300 parser.loads(html)
301 except compat_html_parser.HTMLParseError:
302 pass
303 return parser.get_result()
304
305
306 def clean_html(html):
307 """Clean an HTML snippet into a readable string"""
308 # Newline vs <br />
309 html = html.replace('\n', ' ')
310 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
311 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
312 # Strip html tags
313 html = re.sub('<.*?>', '', html)
314 # Replace html entities
315 html = unescapeHTML(html)
316 return html.strip()
317
318
319 def sanitize_open(filename, open_mode):
320 """Try to open the given filename, and slightly tweak it if this fails.
321
322 Attempts to open the given filename. If this fails, it tries to change
323 the filename slightly, step by step, until it's either able to open it
324 or it fails and raises a final exception, like the standard open()
325 function.
326
327 It returns the tuple (stream, definitive_file_name).
328 """
329 try:
330 if filename == u'-':
331 if sys.platform == 'win32':
332 import msvcrt
333 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
334 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
335 stream = open(encodeFilename(filename), open_mode)
336 return (stream, filename)
337 except (IOError, OSError) as err:
338 if err.errno in (errno.EACCES,):
339 raise
340
341 # In case of error, try to remove win32 forbidden chars
342 alt_filename = os.path.join(
343 re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', path_part)
344 for path_part in os.path.split(filename)
345 )
346 if alt_filename == filename:
347 raise
348 else:
349 # An exception here should be caught in the caller
350 stream = open(encodeFilename(filename), open_mode)
351 return (stream, alt_filename)
352
353
354 def timeconvert(timestr):
355 """Convert RFC 2822 defined time string into system timestamp"""
356 timestamp = None
357 timetuple = email.utils.parsedate_tz(timestr)
358 if timetuple is not None:
359 timestamp = email.utils.mktime_tz(timetuple)
360 return timestamp
361
362 def sanitize_filename(s, restricted=False, is_id=False):
363 """Sanitizes a string so it could be used as part of a filename.
364 If restricted is set, use a stricter subset of allowed characters.
365 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
366 """
367 def replace_insane(char):
368 if char == '?' or ord(char) < 32 or ord(char) == 127:
369 return ''
370 elif char == '"':
371 return '' if restricted else '\''
372 elif char == ':':
373 return '_-' if restricted else ' -'
374 elif char in '\\/|*<>':
375 return '_'
376 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
377 return '_'
378 if restricted and ord(char) > 127:
379 return '_'
380 return char
381
382 result = u''.join(map(replace_insane, s))
383 if not is_id:
384 while '__' in result:
385 result = result.replace('__', '_')
386 result = result.strip('_')
387 # Common case of "Foreign band name - English song title"
388 if restricted and result.startswith('-_'):
389 result = result[2:]
390 if not result:
391 result = '_'
392 return result
393
394 def orderedSet(iterable):
395 """ Remove all duplicates from the input iterable """
396 res = []
397 for el in iterable:
398 if el not in res:
399 res.append(el)
400 return res
401
402 def unescapeHTML(s):
403 """
404 @param s a string
405 """
406 assert type(s) == type(u'')
407
408 result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
409 return result
410
411 def encodeFilename(s):
412 """
413 @param s The name of the file
414 """
415
416 assert type(s) == type(u'')
417
418 # Python 3 has a Unicode API
419 if sys.version_info >= (3, 0):
420 return s
421
422 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
423 # Pass u'' directly to use Unicode APIs on Windows 2000 and up
424 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
425 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
426 return s
427 else:
428 encoding = sys.getfilesystemencoding()
429 if encoding is None:
430 encoding = 'utf-8'
431 return s.encode(encoding, 'ignore')
432
433 def decodeOption(optval):
434 if optval is None:
435 return optval
436 if isinstance(optval, bytes):
437 optval = optval.decode(preferredencoding())
438
439 assert isinstance(optval, compat_str)
440 return optval
441
442 def formatSeconds(secs):
443 if secs > 3600:
444 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
445 elif secs > 60:
446 return '%d:%02d' % (secs // 60, secs % 60)
447 else:
448 return '%d' % secs
449
450 def make_HTTPS_handler(opts):
451 if sys.version_info < (3,2):
452 # Python's 2.x handler is very simplistic
453 return compat_urllib_request.HTTPSHandler()
454 else:
455 import ssl
456 context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
457 context.set_default_verify_paths()
458
459 context.verify_mode = (ssl.CERT_NONE
460 if opts.no_check_certificate
461 else ssl.CERT_REQUIRED)
462 return compat_urllib_request.HTTPSHandler(context=context)
463
464 class ExtractorError(Exception):
465 """Error during info extraction."""
466 def __init__(self, msg, tb=None):
467 """ tb, if given, is the original traceback (so that it can be printed out). """
468 super(ExtractorError, self).__init__(msg)
469 self.traceback = tb
470 self.exc_info = sys.exc_info() # preserve original exception
471
472 def format_traceback(self):
473 if self.traceback is None:
474 return None
475 return u''.join(traceback.format_tb(self.traceback))
476
477
478 class DownloadError(Exception):
479 """Download Error exception.
480
481 This exception may be thrown by FileDownloader objects if they are not
482 configured to continue on errors. They will contain the appropriate
483 error message.
484 """
485 def __init__(self, msg, exc_info=None):
486 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
487 super(DownloadError, self).__init__(msg)
488 self.exc_info = exc_info
489
490
491 class SameFileError(Exception):
492 """Same File exception.
493
494 This exception will be thrown by FileDownloader objects if they detect
495 multiple files would have to be downloaded to the same file on disk.
496 """
497 pass
498
499
500 class PostProcessingError(Exception):
501 """Post Processing exception.
502
503 This exception may be raised by PostProcessor's .run() method to
504 indicate an error in the postprocessing task.
505 """
506 def __init__(self, msg):
507 self.msg = msg
508
509 class MaxDownloadsReached(Exception):
510 """ --max-downloads limit has been reached. """
511 pass
512
513
514 class UnavailableVideoError(Exception):
515 """Unavailable Format exception.
516
517 This exception will be thrown when a video is requested
518 in a format that is not available for that video.
519 """
520 pass
521
522
523 class ContentTooShortError(Exception):
524 """Content Too Short exception.
525
526 This exception may be raised by FileDownloader objects when a file they
527 download is too small for what the server announced first, indicating
528 the connection was probably interrupted.
529 """
530 # Both in bytes
531 downloaded = None
532 expected = None
533
534 def __init__(self, downloaded, expected):
535 self.downloaded = downloaded
536 self.expected = expected
537
538 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
539 """Handler for HTTP requests and responses.
540
541 This class, when installed with an OpenerDirector, automatically adds
542 the standard headers to every HTTP request and handles gzipped and
543 deflated responses from web servers. If compression is to be avoided in
544 a particular request, the original request in the program code only has
545 to include the HTTP header "Youtubedl-No-Compression", which will be
546 removed before making the real request.
547
548 Part of this code was copied from:
549
550 http://techknack.net/python-urllib2-handlers/
551
552 Andrew Rowls, the author of that code, agreed to release it to the
553 public domain.
554 """
555
556 @staticmethod
557 def deflate(data):
558 try:
559 return zlib.decompress(data, -zlib.MAX_WBITS)
560 except zlib.error:
561 return zlib.decompress(data)
562
563 @staticmethod
564 def addinfourl_wrapper(stream, headers, url, code):
565 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
566 return compat_urllib_request.addinfourl(stream, headers, url, code)
567 ret = compat_urllib_request.addinfourl(stream, headers, url)
568 ret.code = code
569 return ret
570
571 def http_request(self, req):
572 for h,v in std_headers.items():
573 if h in req.headers:
574 del req.headers[h]
575 req.add_header(h, v)
576 if 'Youtubedl-no-compression' in req.headers:
577 if 'Accept-encoding' in req.headers:
578 del req.headers['Accept-encoding']
579 del req.headers['Youtubedl-no-compression']
580 if 'Youtubedl-user-agent' in req.headers:
581 if 'User-agent' in req.headers:
582 del req.headers['User-agent']
583 req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
584 del req.headers['Youtubedl-user-agent']
585 return req
586
587 def http_response(self, req, resp):
588 old_resp = resp
589 # gzip
590 if resp.headers.get('Content-encoding', '') == 'gzip':
591 gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
592 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
593 resp.msg = old_resp.msg
594 # deflate
595 if resp.headers.get('Content-encoding', '') == 'deflate':
596 gz = io.BytesIO(self.deflate(resp.read()))
597 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
598 resp.msg = old_resp.msg
599 return resp
600
601 https_request = http_request
602 https_response = http_response
603
604 def unified_strdate(date_str):
605 """Return a string with the date in the format YYYYMMDD"""
606 upload_date = None
607 #Replace commas
608 date_str = date_str.replace(',',' ')
609 # %z (UTC offset) is only supported in python>=3.2
610 date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
611 format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S']
612 for expression in format_expressions:
613 try:
614 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
615 except:
616 pass
617 return upload_date
618
619 def date_from_str(date_str):
620 """
621 Return a datetime object from a string in the format YYYYMMDD or
622 (now|today)[+-][0-9](day|week|month|year)(s)?"""
623 today = datetime.date.today()
624 if date_str == 'now'or date_str == 'today':
625 return today
626 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
627 if match is not None:
628 sign = match.group('sign')
629 time = int(match.group('time'))
630 if sign == '-':
631 time = -time
632 unit = match.group('unit')
633 #A bad aproximation?
634 if unit == 'month':
635 unit = 'day'
636 time *= 30
637 elif unit == 'year':
638 unit = 'day'
639 time *= 365
640 unit += 's'
641 delta = datetime.timedelta(**{unit: time})
642 return today + delta
643 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
644
645 class DateRange(object):
646 """Represents a time interval between two dates"""
647 def __init__(self, start=None, end=None):
648 """start and end must be strings in the format accepted by date"""
649 if start is not None:
650 self.start = date_from_str(start)
651 else:
652 self.start = datetime.datetime.min.date()
653 if end is not None:
654 self.end = date_from_str(end)
655 else:
656 self.end = datetime.datetime.max.date()
657 if self.start > self.end:
658 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
659 @classmethod
660 def day(cls, day):
661 """Returns a range that only contains the given day"""
662 return cls(day,day)
663 def __contains__(self, date):
664 """Check if the date is in the range"""
665 if not isinstance(date, datetime.date):
666 date = date_from_str(date)
667 return self.start <= date <= self.end
668 def __str__(self):
669 return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())