]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/utils.py
debian/watch: Download version specific for distributors.
[youtubedl] / youtube_dl / utils.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 import errno
5 import gzip
6 import io
7 import json
8 import locale
9 import os
10 import re
11 import sys
12 import traceback
13 import zlib
14 import email.utils
15 import json
16 import datetime
17
18 try:
19 import urllib.request as compat_urllib_request
20 except ImportError: # Python 2
21 import urllib2 as compat_urllib_request
22
23 try:
24 import urllib.error as compat_urllib_error
25 except ImportError: # Python 2
26 import urllib2 as compat_urllib_error
27
28 try:
29 import urllib.parse as compat_urllib_parse
30 except ImportError: # Python 2
31 import urllib as compat_urllib_parse
32
33 try:
34 from urllib.parse import urlparse as compat_urllib_parse_urlparse
35 except ImportError: # Python 2
36 from urlparse import urlparse as compat_urllib_parse_urlparse
37
38 try:
39 import http.cookiejar as compat_cookiejar
40 except ImportError: # Python 2
41 import cookielib as compat_cookiejar
42
43 try:
44 import html.entities as compat_html_entities
45 except ImportError: # Python 2
46 import htmlentitydefs as compat_html_entities
47
48 try:
49 import html.parser as compat_html_parser
50 except ImportError: # Python 2
51 import HTMLParser as compat_html_parser
52
53 try:
54 import http.client as compat_http_client
55 except ImportError: # Python 2
56 import httplib as compat_http_client
57
58 try:
59 from subprocess import DEVNULL
60 compat_subprocess_get_DEVNULL = lambda: DEVNULL
61 except ImportError:
62 compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
63
64 try:
65 from urllib.parse import parse_qs as compat_parse_qs
66 except ImportError: # Python 2
67 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
68 # Python 2's version is apparently totally broken
69 def _unquote(string, encoding='utf-8', errors='replace'):
70 if string == '':
71 return string
72 res = string.split('%')
73 if len(res) == 1:
74 return string
75 if encoding is None:
76 encoding = 'utf-8'
77 if errors is None:
78 errors = 'replace'
79 # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
80 pct_sequence = b''
81 string = res[0]
82 for item in res[1:]:
83 try:
84 if not item:
85 raise ValueError
86 pct_sequence += item[:2].decode('hex')
87 rest = item[2:]
88 if not rest:
89 # This segment was just a single percent-encoded character.
90 # May be part of a sequence of code units, so delay decoding.
91 # (Stored in pct_sequence).
92 continue
93 except ValueError:
94 rest = '%' + item
95 # Encountered non-percent-encoded characters. Flush the current
96 # pct_sequence.
97 string += pct_sequence.decode(encoding, errors) + rest
98 pct_sequence = b''
99 if pct_sequence:
100 # Flush the final pct_sequence
101 string += pct_sequence.decode(encoding, errors)
102 return string
103
104 def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
105 encoding='utf-8', errors='replace'):
106 qs, _coerce_result = qs, unicode
107 pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
108 r = []
109 for name_value in pairs:
110 if not name_value and not strict_parsing:
111 continue
112 nv = name_value.split('=', 1)
113 if len(nv) != 2:
114 if strict_parsing:
115 raise ValueError("bad query field: %r" % (name_value,))
116 # Handle case of a control-name with no equal sign
117 if keep_blank_values:
118 nv.append('')
119 else:
120 continue
121 if len(nv[1]) or keep_blank_values:
122 name = nv[0].replace('+', ' ')
123 name = _unquote(name, encoding=encoding, errors=errors)
124 name = _coerce_result(name)
125 value = nv[1].replace('+', ' ')
126 value = _unquote(value, encoding=encoding, errors=errors)
127 value = _coerce_result(value)
128 r.append((name, value))
129 return r
130
131 def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
132 encoding='utf-8', errors='replace'):
133 parsed_result = {}
134 pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
135 encoding=encoding, errors=errors)
136 for name, value in pairs:
137 if name in parsed_result:
138 parsed_result[name].append(value)
139 else:
140 parsed_result[name] = [value]
141 return parsed_result
142
143 try:
144 compat_str = unicode # Python 2
145 except NameError:
146 compat_str = str
147
148 try:
149 compat_chr = unichr # Python 2
150 except NameError:
151 compat_chr = chr
152
153 def compat_ord(c):
154 if type(c) is int: return c
155 else: return ord(c)
156
157 std_headers = {
158 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
159 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
160 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
161 'Accept-Encoding': 'gzip, deflate',
162 'Accept-Language': 'en-us,en;q=0.5',
163 }
164
165 def preferredencoding():
166 """Get preferred encoding.
167
168 Returns the best encoding scheme for the system, based on
169 locale.getpreferredencoding() and some further tweaks.
170 """
171 try:
172 pref = locale.getpreferredencoding()
173 u'TEST'.encode(pref)
174 except:
175 pref = 'UTF-8'
176
177 return pref
178
179 if sys.version_info < (3,0):
180 def compat_print(s):
181 print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
182 else:
183 def compat_print(s):
184 assert type(s) == type(u'')
185 print(s)
186
187 # In Python 2.x, json.dump expects a bytestream.
188 # In Python 3.x, it writes to a character stream
189 if sys.version_info < (3,0):
190 def write_json_file(obj, fn):
191 with open(fn, 'wb') as f:
192 json.dump(obj, f)
193 else:
194 def write_json_file(obj, fn):
195 with open(fn, 'w', encoding='utf-8') as f:
196 json.dump(obj, f)
197
198 def htmlentity_transform(matchobj):
199 """Transforms an HTML entity to a character.
200
201 This function receives a match object and is intended to be used with
202 the re.sub() function.
203 """
204 entity = matchobj.group(1)
205
206 # Known non-numeric HTML entity
207 if entity in compat_html_entities.name2codepoint:
208 return compat_chr(compat_html_entities.name2codepoint[entity])
209
210 mobj = re.match(u'(?u)#(x?\\d+)', entity)
211 if mobj is not None:
212 numstr = mobj.group(1)
213 if numstr.startswith(u'x'):
214 base = 16
215 numstr = u'0%s' % numstr
216 else:
217 base = 10
218 return compat_chr(int(numstr, base))
219
220 # Unknown entity in name, return its literal representation
221 return (u'&%s;' % entity)
222
223 compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
224 class AttrParser(compat_html_parser.HTMLParser):
225 """Modified HTMLParser that isolates a tag with the specified attribute"""
226 def __init__(self, attribute, value):
227 self.attribute = attribute
228 self.value = value
229 self.result = None
230 self.started = False
231 self.depth = {}
232 self.html = None
233 self.watch_startpos = False
234 self.error_count = 0
235 compat_html_parser.HTMLParser.__init__(self)
236
237 def error(self, message):
238 if self.error_count > 10 or self.started:
239 raise compat_html_parser.HTMLParseError(message, self.getpos())
240 self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
241 self.error_count += 1
242 self.goahead(1)
243
244 def loads(self, html):
245 self.html = html
246 self.feed(html)
247 self.close()
248
249 def handle_starttag(self, tag, attrs):
250 attrs = dict(attrs)
251 if self.started:
252 self.find_startpos(None)
253 if self.attribute in attrs and attrs[self.attribute] == self.value:
254 self.result = [tag]
255 self.started = True
256 self.watch_startpos = True
257 if self.started:
258 if not tag in self.depth: self.depth[tag] = 0
259 self.depth[tag] += 1
260
261 def handle_endtag(self, tag):
262 if self.started:
263 if tag in self.depth: self.depth[tag] -= 1
264 if self.depth[self.result[0]] == 0:
265 self.started = False
266 self.result.append(self.getpos())
267
268 def find_startpos(self, x):
269 """Needed to put the start position of the result (self.result[1])
270 after the opening tag with the requested id"""
271 if self.watch_startpos:
272 self.watch_startpos = False
273 self.result.append(self.getpos())
274 handle_entityref = handle_charref = handle_data = handle_comment = \
275 handle_decl = handle_pi = unknown_decl = find_startpos
276
277 def get_result(self):
278 if self.result is None:
279 return None
280 if len(self.result) != 3:
281 return None
282 lines = self.html.split('\n')
283 lines = lines[self.result[1][0]-1:self.result[2][0]]
284 lines[0] = lines[0][self.result[1][1]:]
285 if len(lines) == 1:
286 lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
287 lines[-1] = lines[-1][:self.result[2][1]]
288 return '\n'.join(lines).strip()
289 # Hack for https://github.com/rg3/youtube-dl/issues/662
290 if sys.version_info < (2, 7, 3):
291 AttrParser.parse_endtag = (lambda self, i:
292 i + len("</scr'+'ipt>")
293 if self.rawdata[i:].startswith("</scr'+'ipt>")
294 else compat_html_parser.HTMLParser.parse_endtag(self, i))
295
296 def get_element_by_id(id, html):
297 """Return the content of the tag with the specified ID in the passed HTML document"""
298 return get_element_by_attribute("id", id, html)
299
300 def get_element_by_attribute(attribute, value, html):
301 """Return the content of the tag with the specified attribute in the passed HTML document"""
302 parser = AttrParser(attribute, value)
303 try:
304 parser.loads(html)
305 except compat_html_parser.HTMLParseError:
306 pass
307 return parser.get_result()
308
309
310 def clean_html(html):
311 """Clean an HTML snippet into a readable string"""
312 # Newline vs <br />
313 html = html.replace('\n', ' ')
314 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
315 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
316 # Strip html tags
317 html = re.sub('<.*?>', '', html)
318 # Replace html entities
319 html = unescapeHTML(html)
320 return html.strip()
321
322
323 def sanitize_open(filename, open_mode):
324 """Try to open the given filename, and slightly tweak it if this fails.
325
326 Attempts to open the given filename. If this fails, it tries to change
327 the filename slightly, step by step, until it's either able to open it
328 or it fails and raises a final exception, like the standard open()
329 function.
330
331 It returns the tuple (stream, definitive_file_name).
332 """
333 try:
334 if filename == u'-':
335 if sys.platform == 'win32':
336 import msvcrt
337 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
338 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
339 stream = open(encodeFilename(filename), open_mode)
340 return (stream, filename)
341 except (IOError, OSError) as err:
342 if err.errno in (errno.EACCES,):
343 raise
344
345 # In case of error, try to remove win32 forbidden chars
346 alt_filename = os.path.join(
347 re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', path_part)
348 for path_part in os.path.split(filename)
349 )
350 if alt_filename == filename:
351 raise
352 else:
353 # An exception here should be caught in the caller
354 stream = open(encodeFilename(filename), open_mode)
355 return (stream, alt_filename)
356
357
358 def timeconvert(timestr):
359 """Convert RFC 2822 defined time string into system timestamp"""
360 timestamp = None
361 timetuple = email.utils.parsedate_tz(timestr)
362 if timetuple is not None:
363 timestamp = email.utils.mktime_tz(timetuple)
364 return timestamp
365
366 def sanitize_filename(s, restricted=False, is_id=False):
367 """Sanitizes a string so it could be used as part of a filename.
368 If restricted is set, use a stricter subset of allowed characters.
369 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
370 """
371 def replace_insane(char):
372 if char == '?' or ord(char) < 32 or ord(char) == 127:
373 return ''
374 elif char == '"':
375 return '' if restricted else '\''
376 elif char == ':':
377 return '_-' if restricted else ' -'
378 elif char in '\\/|*<>':
379 return '_'
380 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
381 return '_'
382 if restricted and ord(char) > 127:
383 return '_'
384 return char
385
386 result = u''.join(map(replace_insane, s))
387 if not is_id:
388 while '__' in result:
389 result = result.replace('__', '_')
390 result = result.strip('_')
391 # Common case of "Foreign band name - English song title"
392 if restricted and result.startswith('-_'):
393 result = result[2:]
394 if not result:
395 result = '_'
396 return result
397
398 def orderedSet(iterable):
399 """ Remove all duplicates from the input iterable """
400 res = []
401 for el in iterable:
402 if el not in res:
403 res.append(el)
404 return res
405
406 def unescapeHTML(s):
407 """
408 @param s a string
409 """
410 assert type(s) == type(u'')
411
412 result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
413 return result
414
415 def encodeFilename(s):
416 """
417 @param s The name of the file
418 """
419
420 assert type(s) == type(u'')
421
422 # Python 3 has a Unicode API
423 if sys.version_info >= (3, 0):
424 return s
425
426 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
427 # Pass u'' directly to use Unicode APIs on Windows 2000 and up
428 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
429 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
430 return s
431 else:
432 encoding = sys.getfilesystemencoding()
433 if encoding is None:
434 encoding = 'utf-8'
435 return s.encode(encoding, 'ignore')
436
437 def decodeOption(optval):
438 if optval is None:
439 return optval
440 if isinstance(optval, bytes):
441 optval = optval.decode(preferredencoding())
442
443 assert isinstance(optval, compat_str)
444 return optval
445
446 def formatSeconds(secs):
447 if secs > 3600:
448 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
449 elif secs > 60:
450 return '%d:%02d' % (secs // 60, secs % 60)
451 else:
452 return '%d' % secs
453
454 def make_HTTPS_handler(opts):
455 if sys.version_info < (3,2):
456 # Python's 2.x handler is very simplistic
457 return compat_urllib_request.HTTPSHandler()
458 else:
459 import ssl
460 context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
461 context.set_default_verify_paths()
462
463 context.verify_mode = (ssl.CERT_NONE
464 if opts.no_check_certificate
465 else ssl.CERT_REQUIRED)
466 return compat_urllib_request.HTTPSHandler(context=context)
467
468 class ExtractorError(Exception):
469 """Error during info extraction."""
470 def __init__(self, msg, tb=None):
471 """ tb, if given, is the original traceback (so that it can be printed out). """
472 super(ExtractorError, self).__init__(msg)
473 self.traceback = tb
474 self.exc_info = sys.exc_info() # preserve original exception
475
476 def format_traceback(self):
477 if self.traceback is None:
478 return None
479 return u''.join(traceback.format_tb(self.traceback))
480
481
482 class DownloadError(Exception):
483 """Download Error exception.
484
485 This exception may be thrown by FileDownloader objects if they are not
486 configured to continue on errors. They will contain the appropriate
487 error message.
488 """
489 def __init__(self, msg, exc_info=None):
490 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
491 super(DownloadError, self).__init__(msg)
492 self.exc_info = exc_info
493
494
495 class SameFileError(Exception):
496 """Same File exception.
497
498 This exception will be thrown by FileDownloader objects if they detect
499 multiple files would have to be downloaded to the same file on disk.
500 """
501 pass
502
503
504 class PostProcessingError(Exception):
505 """Post Processing exception.
506
507 This exception may be raised by PostProcessor's .run() method to
508 indicate an error in the postprocessing task.
509 """
510 def __init__(self, msg):
511 self.msg = msg
512
513 class MaxDownloadsReached(Exception):
514 """ --max-downloads limit has been reached. """
515 pass
516
517
518 class UnavailableVideoError(Exception):
519 """Unavailable Format exception.
520
521 This exception will be thrown when a video is requested
522 in a format that is not available for that video.
523 """
524 pass
525
526
527 class ContentTooShortError(Exception):
528 """Content Too Short exception.
529
530 This exception may be raised by FileDownloader objects when a file they
531 download is too small for what the server announced first, indicating
532 the connection was probably interrupted.
533 """
534 # Both in bytes
535 downloaded = None
536 expected = None
537
538 def __init__(self, downloaded, expected):
539 self.downloaded = downloaded
540 self.expected = expected
541
542 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
543 """Handler for HTTP requests and responses.
544
545 This class, when installed with an OpenerDirector, automatically adds
546 the standard headers to every HTTP request and handles gzipped and
547 deflated responses from web servers. If compression is to be avoided in
548 a particular request, the original request in the program code only has
549 to include the HTTP header "Youtubedl-No-Compression", which will be
550 removed before making the real request.
551
552 Part of this code was copied from:
553
554 http://techknack.net/python-urllib2-handlers/
555
556 Andrew Rowls, the author of that code, agreed to release it to the
557 public domain.
558 """
559
560 @staticmethod
561 def deflate(data):
562 try:
563 return zlib.decompress(data, -zlib.MAX_WBITS)
564 except zlib.error:
565 return zlib.decompress(data)
566
567 @staticmethod
568 def addinfourl_wrapper(stream, headers, url, code):
569 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
570 return compat_urllib_request.addinfourl(stream, headers, url, code)
571 ret = compat_urllib_request.addinfourl(stream, headers, url)
572 ret.code = code
573 return ret
574
575 def http_request(self, req):
576 for h,v in std_headers.items():
577 if h in req.headers:
578 del req.headers[h]
579 req.add_header(h, v)
580 if 'Youtubedl-no-compression' in req.headers:
581 if 'Accept-encoding' in req.headers:
582 del req.headers['Accept-encoding']
583 del req.headers['Youtubedl-no-compression']
584 if 'Youtubedl-user-agent' in req.headers:
585 if 'User-agent' in req.headers:
586 del req.headers['User-agent']
587 req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
588 del req.headers['Youtubedl-user-agent']
589 return req
590
591 def http_response(self, req, resp):
592 old_resp = resp
593 # gzip
594 if resp.headers.get('Content-encoding', '') == 'gzip':
595 gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
596 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
597 resp.msg = old_resp.msg
598 # deflate
599 if resp.headers.get('Content-encoding', '') == 'deflate':
600 gz = io.BytesIO(self.deflate(resp.read()))
601 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
602 resp.msg = old_resp.msg
603 return resp
604
605 https_request = http_request
606 https_response = http_response
607
608 def unified_strdate(date_str):
609 """Return a string with the date in the format YYYYMMDD"""
610 upload_date = None
611 #Replace commas
612 date_str = date_str.replace(',',' ')
613 # %z (UTC offset) is only supported in python>=3.2
614 date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
615 format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S']
616 for expression in format_expressions:
617 try:
618 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
619 except:
620 pass
621 return upload_date
622
623 def date_from_str(date_str):
624 """
625 Return a datetime object from a string in the format YYYYMMDD or
626 (now|today)[+-][0-9](day|week|month|year)(s)?"""
627 today = datetime.date.today()
628 if date_str == 'now'or date_str == 'today':
629 return today
630 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
631 if match is not None:
632 sign = match.group('sign')
633 time = int(match.group('time'))
634 if sign == '-':
635 time = -time
636 unit = match.group('unit')
637 #A bad aproximation?
638 if unit == 'month':
639 unit = 'day'
640 time *= 30
641 elif unit == 'year':
642 unit = 'day'
643 time *= 365
644 unit += 's'
645 delta = datetime.timedelta(**{unit: time})
646 return today + delta
647 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
648
649 class DateRange(object):
650 """Represents a time interval between two dates"""
651 def __init__(self, start=None, end=None):
652 """start and end must be strings in the format accepted by date"""
653 if start is not None:
654 self.start = date_from_str(start)
655 else:
656 self.start = datetime.datetime.min.date()
657 if end is not None:
658 self.end = date_from_str(end)
659 else:
660 self.end = datetime.datetime.max.date()
661 if self.start > self.end:
662 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
663 @classmethod
664 def day(cls, day):
665 """Returns a range that only contains the given day"""
666 return cls(day,day)
667 def __contains__(self, date):
668 """Check if the date is in the range"""
669 if not isinstance(date, datetime.date):
670 date = date_from_str(date)
671 return self.start <= date <= self.end
672 def __str__(self):
673 return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())