]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/utils.py
d7b737e21639679fe665d22e6ca4089f5c791618
[youtubedl] / youtube_dl / utils.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import unicode_literals
5
6 import base64
7 import calendar
8 import codecs
9 import contextlib
10 import ctypes
11 import datetime
12 import email.utils
13 import errno
14 import functools
15 import gzip
16 import itertools
17 import io
18 import json
19 import locale
20 import math
21 import operator
22 import os
23 import pipes
24 import platform
25 import re
26 import ssl
27 import socket
28 import struct
29 import subprocess
30 import sys
31 import tempfile
32 import traceback
33 import xml.etree.ElementTree
34 import zlib
35
36 from .compat import (
37 compat_basestring,
38 compat_chr,
39 compat_etree_fromstring,
40 compat_html_entities,
41 compat_http_client,
42 compat_kwargs,
43 compat_parse_qs,
44 compat_socket_create_connection,
45 compat_str,
46 compat_urllib_error,
47 compat_urllib_parse,
48 compat_urllib_parse_urlparse,
49 compat_urllib_request,
50 compat_urlparse,
51 shlex_quote,
52 )
53
54
55 # This is not clearly defined otherwise
56 compiled_regex_type = type(re.compile(''))
57
58 std_headers = {
59 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
60 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
61 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
62 'Accept-Encoding': 'gzip, deflate',
63 'Accept-Language': 'en-us,en;q=0.5',
64 }
65
66
67 NO_DEFAULT = object()
68
69 ENGLISH_MONTH_NAMES = [
70 'January', 'February', 'March', 'April', 'May', 'June',
71 'July', 'August', 'September', 'October', 'November', 'December']
72
73
74 def preferredencoding():
75 """Get preferred encoding.
76
77 Returns the best encoding scheme for the system, based on
78 locale.getpreferredencoding() and some further tweaks.
79 """
80 try:
81 pref = locale.getpreferredencoding()
82 'TEST'.encode(pref)
83 except Exception:
84 pref = 'UTF-8'
85
86 return pref
87
88
89 def write_json_file(obj, fn):
90 """ Encode obj as JSON and write it to fn, atomically if possible """
91
92 fn = encodeFilename(fn)
93 if sys.version_info < (3, 0) and sys.platform != 'win32':
94 encoding = get_filesystem_encoding()
95 # os.path.basename returns a bytes object, but NamedTemporaryFile
96 # will fail if the filename contains non ascii characters unless we
97 # use a unicode object
98 path_basename = lambda f: os.path.basename(fn).decode(encoding)
99 # the same for os.path.dirname
100 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
101 else:
102 path_basename = os.path.basename
103 path_dirname = os.path.dirname
104
105 args = {
106 'suffix': '.tmp',
107 'prefix': path_basename(fn) + '.',
108 'dir': path_dirname(fn),
109 'delete': False,
110 }
111
112 # In Python 2.x, json.dump expects a bytestream.
113 # In Python 3.x, it writes to a character stream
114 if sys.version_info < (3, 0):
115 args['mode'] = 'wb'
116 else:
117 args.update({
118 'mode': 'w',
119 'encoding': 'utf-8',
120 })
121
122 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
123
124 try:
125 with tf:
126 json.dump(obj, tf)
127 if sys.platform == 'win32':
128 # Need to remove existing file on Windows, else os.rename raises
129 # WindowsError or FileExistsError.
130 try:
131 os.unlink(fn)
132 except OSError:
133 pass
134 os.rename(tf.name, fn)
135 except Exception:
136 try:
137 os.remove(tf.name)
138 except OSError:
139 pass
140 raise
141
142
143 if sys.version_info >= (2, 7):
144 def find_xpath_attr(node, xpath, key, val=None):
145 """ Find the xpath xpath[@key=val] """
146 assert re.match(r'^[a-zA-Z_-]+$', key)
147 if val:
148 assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
149 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
150 return node.find(expr)
151 else:
152 def find_xpath_attr(node, xpath, key, val=None):
153 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
154 # .//node does not match if a node is a direct child of . !
155 if isinstance(xpath, compat_str):
156 xpath = xpath.encode('ascii')
157
158 for f in node.findall(xpath):
159 if key not in f.attrib:
160 continue
161 if val is None or f.attrib.get(key) == val:
162 return f
163 return None
164
165 # On python2.6 the xml.etree.ElementTree.Element methods don't support
166 # the namespace parameter
167
168
169 def xpath_with_ns(path, ns_map):
170 components = [c.split(':') for c in path.split('/')]
171 replaced = []
172 for c in components:
173 if len(c) == 1:
174 replaced.append(c[0])
175 else:
176 ns, tag = c
177 replaced.append('{%s}%s' % (ns_map[ns], tag))
178 return '/'.join(replaced)
179
180
181 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
182 def _find_xpath(xpath):
183 if sys.version_info < (2, 7): # Crazy 2.6
184 xpath = xpath.encode('ascii')
185 return node.find(xpath)
186
187 if isinstance(xpath, (str, compat_str)):
188 n = _find_xpath(xpath)
189 else:
190 for xp in xpath:
191 n = _find_xpath(xp)
192 if n is not None:
193 break
194
195 if n is None:
196 if default is not NO_DEFAULT:
197 return default
198 elif fatal:
199 name = xpath if name is None else name
200 raise ExtractorError('Could not find XML element %s' % name)
201 else:
202 return None
203 return n
204
205
206 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
207 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
208 if n is None or n == default:
209 return n
210 if n.text is None:
211 if default is not NO_DEFAULT:
212 return default
213 elif fatal:
214 name = xpath if name is None else name
215 raise ExtractorError('Could not find XML element\'s text %s' % name)
216 else:
217 return None
218 return n.text
219
220
221 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
222 n = find_xpath_attr(node, xpath, key)
223 if n is None:
224 if default is not NO_DEFAULT:
225 return default
226 elif fatal:
227 name = '%s[@%s]' % (xpath, key) if name is None else name
228 raise ExtractorError('Could not find XML attribute %s' % name)
229 else:
230 return None
231 return n.attrib[key]
232
233
234 def get_element_by_id(id, html):
235 """Return the content of the tag with the specified ID in the passed HTML document"""
236 return get_element_by_attribute("id", id, html)
237
238
239 def get_element_by_attribute(attribute, value, html):
240 """Return the content of the tag with the specified attribute in the passed HTML document"""
241
242 m = re.search(r'''(?xs)
243 <([a-zA-Z0-9:._-]+)
244 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
245 \s+%s=['"]?%s['"]?
246 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
247 \s*>
248 (?P<content>.*?)
249 </\1>
250 ''' % (re.escape(attribute), re.escape(value)), html)
251
252 if not m:
253 return None
254 res = m.group('content')
255
256 if res.startswith('"') or res.startswith("'"):
257 res = res[1:-1]
258
259 return unescapeHTML(res)
260
261
262 def clean_html(html):
263 """Clean an HTML snippet into a readable string"""
264
265 if html is None: # Convenience for sanitizing descriptions etc.
266 return html
267
268 # Newline vs <br />
269 html = html.replace('\n', ' ')
270 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
271 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
272 # Strip html tags
273 html = re.sub('<.*?>', '', html)
274 # Replace html entities
275 html = unescapeHTML(html)
276 return html.strip()
277
278
279 def sanitize_open(filename, open_mode):
280 """Try to open the given filename, and slightly tweak it if this fails.
281
282 Attempts to open the given filename. If this fails, it tries to change
283 the filename slightly, step by step, until it's either able to open it
284 or it fails and raises a final exception, like the standard open()
285 function.
286
287 It returns the tuple (stream, definitive_file_name).
288 """
289 try:
290 if filename == '-':
291 if sys.platform == 'win32':
292 import msvcrt
293 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
294 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
295 stream = open(encodeFilename(filename), open_mode)
296 return (stream, filename)
297 except (IOError, OSError) as err:
298 if err.errno in (errno.EACCES,):
299 raise
300
301 # In case of error, try to remove win32 forbidden chars
302 alt_filename = sanitize_path(filename)
303 if alt_filename == filename:
304 raise
305 else:
306 # An exception here should be caught in the caller
307 stream = open(encodeFilename(alt_filename), open_mode)
308 return (stream, alt_filename)
309
310
311 def timeconvert(timestr):
312 """Convert RFC 2822 defined time string into system timestamp"""
313 timestamp = None
314 timetuple = email.utils.parsedate_tz(timestr)
315 if timetuple is not None:
316 timestamp = email.utils.mktime_tz(timetuple)
317 return timestamp
318
319
320 def sanitize_filename(s, restricted=False, is_id=False):
321 """Sanitizes a string so it could be used as part of a filename.
322 If restricted is set, use a stricter subset of allowed characters.
323 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
324 """
325 def replace_insane(char):
326 if char == '?' or ord(char) < 32 or ord(char) == 127:
327 return ''
328 elif char == '"':
329 return '' if restricted else '\''
330 elif char == ':':
331 return '_-' if restricted else ' -'
332 elif char in '\\/|*<>':
333 return '_'
334 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
335 return '_'
336 if restricted and ord(char) > 127:
337 return '_'
338 return char
339
340 # Handle timestamps
341 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
342 result = ''.join(map(replace_insane, s))
343 if not is_id:
344 while '__' in result:
345 result = result.replace('__', '_')
346 result = result.strip('_')
347 # Common case of "Foreign band name - English song title"
348 if restricted and result.startswith('-_'):
349 result = result[2:]
350 if result.startswith('-'):
351 result = '_' + result[len('-'):]
352 result = result.lstrip('.')
353 if not result:
354 result = '_'
355 return result
356
357
358 def sanitize_path(s):
359 """Sanitizes and normalizes path on Windows"""
360 if sys.platform != 'win32':
361 return s
362 drive_or_unc, _ = os.path.splitdrive(s)
363 if sys.version_info < (2, 7) and not drive_or_unc:
364 drive_or_unc, _ = os.path.splitunc(s)
365 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
366 if drive_or_unc:
367 norm_path.pop(0)
368 sanitized_path = [
369 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
370 for path_part in norm_path]
371 if drive_or_unc:
372 sanitized_path.insert(0, drive_or_unc + os.path.sep)
373 return os.path.join(*sanitized_path)
374
375
376 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
377 # unwanted failures due to missing protocol
378 def sanitized_Request(url, *args, **kwargs):
379 return compat_urllib_request.Request(
380 'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
381
382
383 def orderedSet(iterable):
384 """ Remove all duplicates from the input iterable """
385 res = []
386 for el in iterable:
387 if el not in res:
388 res.append(el)
389 return res
390
391
392 def _htmlentity_transform(entity):
393 """Transforms an HTML entity to a character."""
394 # Known non-numeric HTML entity
395 if entity in compat_html_entities.name2codepoint:
396 return compat_chr(compat_html_entities.name2codepoint[entity])
397
398 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
399 if mobj is not None:
400 numstr = mobj.group(1)
401 if numstr.startswith('x'):
402 base = 16
403 numstr = '0%s' % numstr
404 else:
405 base = 10
406 # See https://github.com/rg3/youtube-dl/issues/7518
407 try:
408 return compat_chr(int(numstr, base))
409 except ValueError:
410 pass
411
412 # Unknown entity in name, return its literal representation
413 return '&%s;' % entity
414
415
416 def unescapeHTML(s):
417 if s is None:
418 return None
419 assert type(s) == compat_str
420
421 return re.sub(
422 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
423
424
425 def get_subprocess_encoding():
426 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
427 # For subprocess calls, encode with locale encoding
428 # Refer to http://stackoverflow.com/a/9951851/35070
429 encoding = preferredencoding()
430 else:
431 encoding = sys.getfilesystemencoding()
432 if encoding is None:
433 encoding = 'utf-8'
434 return encoding
435
436
437 def encodeFilename(s, for_subprocess=False):
438 """
439 @param s The name of the file
440 """
441
442 assert type(s) == compat_str
443
444 # Python 3 has a Unicode API
445 if sys.version_info >= (3, 0):
446 return s
447
448 # Pass '' directly to use Unicode APIs on Windows 2000 and up
449 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
450 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
451 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
452 return s
453
454 return s.encode(get_subprocess_encoding(), 'ignore')
455
456
457 def decodeFilename(b, for_subprocess=False):
458
459 if sys.version_info >= (3, 0):
460 return b
461
462 if not isinstance(b, bytes):
463 return b
464
465 return b.decode(get_subprocess_encoding(), 'ignore')
466
467
468 def encodeArgument(s):
469 if not isinstance(s, compat_str):
470 # Legacy code that uses byte strings
471 # Uncomment the following line after fixing all post processors
472 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
473 s = s.decode('ascii')
474 return encodeFilename(s, True)
475
476
477 def decodeArgument(b):
478 return decodeFilename(b, True)
479
480
481 def decodeOption(optval):
482 if optval is None:
483 return optval
484 if isinstance(optval, bytes):
485 optval = optval.decode(preferredencoding())
486
487 assert isinstance(optval, compat_str)
488 return optval
489
490
491 def formatSeconds(secs):
492 if secs > 3600:
493 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
494 elif secs > 60:
495 return '%d:%02d' % (secs // 60, secs % 60)
496 else:
497 return '%d' % secs
498
499
500 def make_HTTPS_handler(params, **kwargs):
501 opts_no_check_certificate = params.get('nocheckcertificate', False)
502 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
503 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
504 if opts_no_check_certificate:
505 context.check_hostname = False
506 context.verify_mode = ssl.CERT_NONE
507 try:
508 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
509 except TypeError:
510 # Python 2.7.8
511 # (create_default_context present but HTTPSHandler has no context=)
512 pass
513
514 if sys.version_info < (3, 2):
515 return YoutubeDLHTTPSHandler(params, **kwargs)
516 else: # Python < 3.4
517 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
518 context.verify_mode = (ssl.CERT_NONE
519 if opts_no_check_certificate
520 else ssl.CERT_REQUIRED)
521 context.set_default_verify_paths()
522 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
523
524
525 def bug_reports_message():
526 if ytdl_is_updateable():
527 update_cmd = 'type youtube-dl -U to update'
528 else:
529 update_cmd = 'see https://yt-dl.org/update on how to update'
530 msg = '; please report this issue on https://yt-dl.org/bug .'
531 msg += ' Make sure you are using the latest version; %s.' % update_cmd
532 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
533 return msg
534
535
536 class ExtractorError(Exception):
537 """Error during info extraction."""
538
539 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
540 """ tb, if given, is the original traceback (so that it can be printed out).
541 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
542 """
543
544 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
545 expected = True
546 if video_id is not None:
547 msg = video_id + ': ' + msg
548 if cause:
549 msg += ' (caused by %r)' % cause
550 if not expected:
551 msg += bug_reports_message()
552 super(ExtractorError, self).__init__(msg)
553
554 self.traceback = tb
555 self.exc_info = sys.exc_info() # preserve original exception
556 self.cause = cause
557 self.video_id = video_id
558
559 def format_traceback(self):
560 if self.traceback is None:
561 return None
562 return ''.join(traceback.format_tb(self.traceback))
563
564
565 class UnsupportedError(ExtractorError):
566 def __init__(self, url):
567 super(UnsupportedError, self).__init__(
568 'Unsupported URL: %s' % url, expected=True)
569 self.url = url
570
571
572 class RegexNotFoundError(ExtractorError):
573 """Error when a regex didn't match"""
574 pass
575
576
577 class DownloadError(Exception):
578 """Download Error exception.
579
580 This exception may be thrown by FileDownloader objects if they are not
581 configured to continue on errors. They will contain the appropriate
582 error message.
583 """
584
585 def __init__(self, msg, exc_info=None):
586 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
587 super(DownloadError, self).__init__(msg)
588 self.exc_info = exc_info
589
590
591 class SameFileError(Exception):
592 """Same File exception.
593
594 This exception will be thrown by FileDownloader objects if they detect
595 multiple files would have to be downloaded to the same file on disk.
596 """
597 pass
598
599
600 class PostProcessingError(Exception):
601 """Post Processing exception.
602
603 This exception may be raised by PostProcessor's .run() method to
604 indicate an error in the postprocessing task.
605 """
606
607 def __init__(self, msg):
608 self.msg = msg
609
610
611 class MaxDownloadsReached(Exception):
612 """ --max-downloads limit has been reached. """
613 pass
614
615
616 class UnavailableVideoError(Exception):
617 """Unavailable Format exception.
618
619 This exception will be thrown when a video is requested
620 in a format that is not available for that video.
621 """
622 pass
623
624
625 class ContentTooShortError(Exception):
626 """Content Too Short exception.
627
628 This exception may be raised by FileDownloader objects when a file they
629 download is too small for what the server announced first, indicating
630 the connection was probably interrupted.
631 """
632
633 def __init__(self, downloaded, expected):
634 # Both in bytes
635 self.downloaded = downloaded
636 self.expected = expected
637
638
639 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
640 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
641 # expected HTTP responses to meet HTTP/1.0 or later (see also
642 # https://github.com/rg3/youtube-dl/issues/6727)
643 if sys.version_info < (3, 0):
644 kwargs[b'strict'] = True
645 hc = http_class(*args, **kwargs)
646 source_address = ydl_handler._params.get('source_address')
647 if source_address is not None:
648 sa = (source_address, 0)
649 if hasattr(hc, 'source_address'): # Python 2.7+
650 hc.source_address = sa
651 else: # Python 2.6
652 def _hc_connect(self, *args, **kwargs):
653 sock = compat_socket_create_connection(
654 (self.host, self.port), self.timeout, sa)
655 if is_https:
656 self.sock = ssl.wrap_socket(
657 sock, self.key_file, self.cert_file,
658 ssl_version=ssl.PROTOCOL_TLSv1)
659 else:
660 self.sock = sock
661 hc.connect = functools.partial(_hc_connect, hc)
662
663 return hc
664
665
666 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
667 """Handler for HTTP requests and responses.
668
669 This class, when installed with an OpenerDirector, automatically adds
670 the standard headers to every HTTP request and handles gzipped and
671 deflated responses from web servers. If compression is to be avoided in
672 a particular request, the original request in the program code only has
673 to include the HTTP header "Youtubedl-No-Compression", which will be
674 removed before making the real request.
675
676 Part of this code was copied from:
677
678 http://techknack.net/python-urllib2-handlers/
679
680 Andrew Rowls, the author of that code, agreed to release it to the
681 public domain.
682 """
683
684 def __init__(self, params, *args, **kwargs):
685 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
686 self._params = params
687
688 def http_open(self, req):
689 return self.do_open(functools.partial(
690 _create_http_connection, self, compat_http_client.HTTPConnection, False),
691 req)
692
693 @staticmethod
694 def deflate(data):
695 try:
696 return zlib.decompress(data, -zlib.MAX_WBITS)
697 except zlib.error:
698 return zlib.decompress(data)
699
700 @staticmethod
701 def addinfourl_wrapper(stream, headers, url, code):
702 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
703 return compat_urllib_request.addinfourl(stream, headers, url, code)
704 ret = compat_urllib_request.addinfourl(stream, headers, url)
705 ret.code = code
706 return ret
707
708 def http_request(self, req):
709 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
710 # always respected by websites, some tend to give out URLs with non percent-encoded
711 # non-ASCII characters (see telemb.py, ard.py [#3412])
712 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
713 # To work around aforementioned issue we will replace request's original URL with
714 # percent-encoded one
715 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
716 # the code of this workaround has been moved here from YoutubeDL.urlopen()
717 url = req.get_full_url()
718 url_escaped = escape_url(url)
719
720 # Substitute URL if any change after escaping
721 if url != url_escaped:
722 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
723 new_req = req_type(
724 url_escaped, data=req.data, headers=req.headers,
725 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
726 new_req.timeout = req.timeout
727 req = new_req
728
729 for h, v in std_headers.items():
730 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
731 # The dict keys are capitalized because of this bug by urllib
732 if h.capitalize() not in req.headers:
733 req.add_header(h, v)
734 if 'Youtubedl-no-compression' in req.headers:
735 if 'Accept-encoding' in req.headers:
736 del req.headers['Accept-encoding']
737 del req.headers['Youtubedl-no-compression']
738
739 if sys.version_info < (2, 7) and '#' in req.get_full_url():
740 # Python 2.6 is brain-dead when it comes to fragments
741 req._Request__original = req._Request__original.partition('#')[0]
742 req._Request__r_type = req._Request__r_type.partition('#')[0]
743
744 return req
745
746 def http_response(self, req, resp):
747 old_resp = resp
748 # gzip
749 if resp.headers.get('Content-encoding', '') == 'gzip':
750 content = resp.read()
751 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
752 try:
753 uncompressed = io.BytesIO(gz.read())
754 except IOError as original_ioerror:
755 # There may be junk add the end of the file
756 # See http://stackoverflow.com/q/4928560/35070 for details
757 for i in range(1, 1024):
758 try:
759 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
760 uncompressed = io.BytesIO(gz.read())
761 except IOError:
762 continue
763 break
764 else:
765 raise original_ioerror
766 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
767 resp.msg = old_resp.msg
768 # deflate
769 if resp.headers.get('Content-encoding', '') == 'deflate':
770 gz = io.BytesIO(self.deflate(resp.read()))
771 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
772 resp.msg = old_resp.msg
773 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
774 # https://github.com/rg3/youtube-dl/issues/6457).
775 if 300 <= resp.code < 400:
776 location = resp.headers.get('Location')
777 if location:
778 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
779 if sys.version_info >= (3, 0):
780 location = location.encode('iso-8859-1').decode('utf-8')
781 location_escaped = escape_url(location)
782 if location != location_escaped:
783 del resp.headers['Location']
784 resp.headers['Location'] = location_escaped
785 return resp
786
787 https_request = http_request
788 https_response = http_response
789
790
791 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
792 def __init__(self, params, https_conn_class=None, *args, **kwargs):
793 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
794 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
795 self._params = params
796
797 def https_open(self, req):
798 kwargs = {}
799 if hasattr(self, '_context'): # python > 2.6
800 kwargs['context'] = self._context
801 if hasattr(self, '_check_hostname'): # python 3.x
802 kwargs['check_hostname'] = self._check_hostname
803 return self.do_open(functools.partial(
804 _create_http_connection, self, self._https_conn_class, True),
805 req, **kwargs)
806
807
808 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
809 def __init__(self, cookiejar=None):
810 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
811
812 def http_response(self, request, response):
813 # Python 2 will choke on next HTTP request in row if there are non-ASCII
814 # characters in Set-Cookie HTTP header of last response (see
815 # https://github.com/rg3/youtube-dl/issues/6769).
816 # In order to at least prevent crashing we will percent encode Set-Cookie
817 # header before HTTPCookieProcessor starts processing it.
818 # if sys.version_info < (3, 0) and response.headers:
819 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
820 # set_cookie = response.headers.get(set_cookie_header)
821 # if set_cookie:
822 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
823 # if set_cookie != set_cookie_escaped:
824 # del response.headers[set_cookie_header]
825 # response.headers[set_cookie_header] = set_cookie_escaped
826 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
827
828 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
829 https_response = http_response
830
831
832 def parse_iso8601(date_str, delimiter='T', timezone=None):
833 """ Return a UNIX timestamp from the given date """
834
835 if date_str is None:
836 return None
837
838 date_str = re.sub(r'\.[0-9]+', '', date_str)
839
840 if timezone is None:
841 m = re.search(
842 r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
843 date_str)
844 if not m:
845 timezone = datetime.timedelta()
846 else:
847 date_str = date_str[:-len(m.group(0))]
848 if not m.group('sign'):
849 timezone = datetime.timedelta()
850 else:
851 sign = 1 if m.group('sign') == '+' else -1
852 timezone = datetime.timedelta(
853 hours=sign * int(m.group('hours')),
854 minutes=sign * int(m.group('minutes')))
855 try:
856 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
857 dt = datetime.datetime.strptime(date_str, date_format) - timezone
858 return calendar.timegm(dt.timetuple())
859 except ValueError:
860 pass
861
862
863 def unified_strdate(date_str, day_first=True):
864 """Return a string with the date in the format YYYYMMDD"""
865
866 if date_str is None:
867 return None
868 upload_date = None
869 # Replace commas
870 date_str = date_str.replace(',', ' ')
871 # %z (UTC offset) is only supported in python>=3.2
872 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
873 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
874 # Remove AM/PM + timezone
875 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
876
877 format_expressions = [
878 '%d %B %Y',
879 '%d %b %Y',
880 '%B %d %Y',
881 '%b %d %Y',
882 '%b %dst %Y %I:%M%p',
883 '%b %dnd %Y %I:%M%p',
884 '%b %dth %Y %I:%M%p',
885 '%Y %m %d',
886 '%Y-%m-%d',
887 '%Y/%m/%d',
888 '%Y/%m/%d %H:%M:%S',
889 '%Y-%m-%d %H:%M:%S',
890 '%Y-%m-%d %H:%M:%S.%f',
891 '%d.%m.%Y %H:%M',
892 '%d.%m.%Y %H.%M',
893 '%Y-%m-%dT%H:%M:%SZ',
894 '%Y-%m-%dT%H:%M:%S.%fZ',
895 '%Y-%m-%dT%H:%M:%S.%f0Z',
896 '%Y-%m-%dT%H:%M:%S',
897 '%Y-%m-%dT%H:%M:%S.%f',
898 '%Y-%m-%dT%H:%M',
899 ]
900 if day_first:
901 format_expressions.extend([
902 '%d-%m-%Y',
903 '%d.%m.%Y',
904 '%d/%m/%Y',
905 '%d/%m/%y',
906 '%d/%m/%Y %H:%M:%S',
907 ])
908 else:
909 format_expressions.extend([
910 '%m-%d-%Y',
911 '%m.%d.%Y',
912 '%m/%d/%Y',
913 '%m/%d/%y',
914 '%m/%d/%Y %H:%M:%S',
915 ])
916 for expression in format_expressions:
917 try:
918 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
919 except ValueError:
920 pass
921 if upload_date is None:
922 timetuple = email.utils.parsedate_tz(date_str)
923 if timetuple:
924 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
925 if upload_date is not None:
926 return compat_str(upload_date)
927
928
929 def determine_ext(url, default_ext='unknown_video'):
930 if url is None:
931 return default_ext
932 guess = url.partition('?')[0].rpartition('.')[2]
933 if re.match(r'^[A-Za-z0-9]+$', guess):
934 return guess
935 elif guess.rstrip('/') in (
936 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
937 'flv', 'f4v', 'f4a', 'f4b',
938 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
939 'mkv', 'mka', 'mk3d',
940 'avi', 'divx',
941 'mov',
942 'asf', 'wmv', 'wma',
943 '3gp', '3g2',
944 'mp3',
945 'flac',
946 'ape',
947 'wav',
948 'f4f', 'f4m', 'm3u8', 'smil'):
949 return guess.rstrip('/')
950 else:
951 return default_ext
952
953
954 def subtitles_filename(filename, sub_lang, sub_format):
955 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
956
957
958 def date_from_str(date_str):
959 """
960 Return a datetime object from a string in the format YYYYMMDD or
961 (now|today)[+-][0-9](day|week|month|year)(s)?"""
962 today = datetime.date.today()
963 if date_str in ('now', 'today'):
964 return today
965 if date_str == 'yesterday':
966 return today - datetime.timedelta(days=1)
967 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
968 if match is not None:
969 sign = match.group('sign')
970 time = int(match.group('time'))
971 if sign == '-':
972 time = -time
973 unit = match.group('unit')
974 # A bad aproximation?
975 if unit == 'month':
976 unit = 'day'
977 time *= 30
978 elif unit == 'year':
979 unit = 'day'
980 time *= 365
981 unit += 's'
982 delta = datetime.timedelta(**{unit: time})
983 return today + delta
984 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
985
986
987 def hyphenate_date(date_str):
988 """
989 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
990 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
991 if match is not None:
992 return '-'.join(match.groups())
993 else:
994 return date_str
995
996
997 class DateRange(object):
998 """Represents a time interval between two dates"""
999
1000 def __init__(self, start=None, end=None):
1001 """start and end must be strings in the format accepted by date"""
1002 if start is not None:
1003 self.start = date_from_str(start)
1004 else:
1005 self.start = datetime.datetime.min.date()
1006 if end is not None:
1007 self.end = date_from_str(end)
1008 else:
1009 self.end = datetime.datetime.max.date()
1010 if self.start > self.end:
1011 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1012
1013 @classmethod
1014 def day(cls, day):
1015 """Returns a range that only contains the given day"""
1016 return cls(day, day)
1017
1018 def __contains__(self, date):
1019 """Check if the date is in the range"""
1020 if not isinstance(date, datetime.date):
1021 date = date_from_str(date)
1022 return self.start <= date <= self.end
1023
1024 def __str__(self):
1025 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1026
1027
1028 def platform_name():
1029 """ Returns the platform name as a compat_str """
1030 res = platform.platform()
1031 if isinstance(res, bytes):
1032 res = res.decode(preferredencoding())
1033
1034 assert isinstance(res, compat_str)
1035 return res
1036
1037
1038 def _windows_write_string(s, out):
1039 """ Returns True if the string was written using special methods,
1040 False if it has yet to be written out."""
1041 # Adapted from http://stackoverflow.com/a/3259271/35070
1042
1043 import ctypes
1044 import ctypes.wintypes
1045
1046 WIN_OUTPUT_IDS = {
1047 1: -11,
1048 2: -12,
1049 }
1050
1051 try:
1052 fileno = out.fileno()
1053 except AttributeError:
1054 # If the output stream doesn't have a fileno, it's virtual
1055 return False
1056 except io.UnsupportedOperation:
1057 # Some strange Windows pseudo files?
1058 return False
1059 if fileno not in WIN_OUTPUT_IDS:
1060 return False
1061
1062 GetStdHandle = ctypes.WINFUNCTYPE(
1063 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1064 (b"GetStdHandle", ctypes.windll.kernel32))
1065 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1066
1067 WriteConsoleW = ctypes.WINFUNCTYPE(
1068 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1069 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1070 ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
1071 written = ctypes.wintypes.DWORD(0)
1072
1073 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
1074 FILE_TYPE_CHAR = 0x0002
1075 FILE_TYPE_REMOTE = 0x8000
1076 GetConsoleMode = ctypes.WINFUNCTYPE(
1077 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1078 ctypes.POINTER(ctypes.wintypes.DWORD))(
1079 (b"GetConsoleMode", ctypes.windll.kernel32))
1080 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1081
1082 def not_a_console(handle):
1083 if handle == INVALID_HANDLE_VALUE or handle is None:
1084 return True
1085 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1086 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1087
1088 if not_a_console(h):
1089 return False
1090
1091 def next_nonbmp_pos(s):
1092 try:
1093 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1094 except StopIteration:
1095 return len(s)
1096
1097 while s:
1098 count = min(next_nonbmp_pos(s), 1024)
1099
1100 ret = WriteConsoleW(
1101 h, s, count if count else 2, ctypes.byref(written), None)
1102 if ret == 0:
1103 raise OSError('Failed to write string')
1104 if not count: # We just wrote a non-BMP character
1105 assert written.value == 2
1106 s = s[1:]
1107 else:
1108 assert written.value > 0
1109 s = s[written.value:]
1110 return True
1111
1112
1113 def write_string(s, out=None, encoding=None):
1114 if out is None:
1115 out = sys.stderr
1116 assert type(s) == compat_str
1117
1118 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1119 if _windows_write_string(s, out):
1120 return
1121
1122 if ('b' in getattr(out, 'mode', '') or
1123 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1124 byt = s.encode(encoding or preferredencoding(), 'ignore')
1125 out.write(byt)
1126 elif hasattr(out, 'buffer'):
1127 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1128 byt = s.encode(enc, 'ignore')
1129 out.buffer.write(byt)
1130 else:
1131 out.write(s)
1132 out.flush()
1133
1134
1135 def bytes_to_intlist(bs):
1136 if not bs:
1137 return []
1138 if isinstance(bs[0], int): # Python 3
1139 return list(bs)
1140 else:
1141 return [ord(c) for c in bs]
1142
1143
1144 def intlist_to_bytes(xs):
1145 if not xs:
1146 return b''
1147 return struct_pack('%dB' % len(xs), *xs)
1148
1149
1150 # Cross-platform file locking
1151 if sys.platform == 'win32':
1152 import ctypes.wintypes
1153 import msvcrt
1154
1155 class OVERLAPPED(ctypes.Structure):
1156 _fields_ = [
1157 ('Internal', ctypes.wintypes.LPVOID),
1158 ('InternalHigh', ctypes.wintypes.LPVOID),
1159 ('Offset', ctypes.wintypes.DWORD),
1160 ('OffsetHigh', ctypes.wintypes.DWORD),
1161 ('hEvent', ctypes.wintypes.HANDLE),
1162 ]
1163
1164 kernel32 = ctypes.windll.kernel32
1165 LockFileEx = kernel32.LockFileEx
1166 LockFileEx.argtypes = [
1167 ctypes.wintypes.HANDLE, # hFile
1168 ctypes.wintypes.DWORD, # dwFlags
1169 ctypes.wintypes.DWORD, # dwReserved
1170 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1171 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1172 ctypes.POINTER(OVERLAPPED) # Overlapped
1173 ]
1174 LockFileEx.restype = ctypes.wintypes.BOOL
1175 UnlockFileEx = kernel32.UnlockFileEx
1176 UnlockFileEx.argtypes = [
1177 ctypes.wintypes.HANDLE, # hFile
1178 ctypes.wintypes.DWORD, # dwReserved
1179 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1180 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1181 ctypes.POINTER(OVERLAPPED) # Overlapped
1182 ]
1183 UnlockFileEx.restype = ctypes.wintypes.BOOL
1184 whole_low = 0xffffffff
1185 whole_high = 0x7fffffff
1186
1187 def _lock_file(f, exclusive):
1188 overlapped = OVERLAPPED()
1189 overlapped.Offset = 0
1190 overlapped.OffsetHigh = 0
1191 overlapped.hEvent = 0
1192 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1193 handle = msvcrt.get_osfhandle(f.fileno())
1194 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1195 whole_low, whole_high, f._lock_file_overlapped_p):
1196 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1197
1198 def _unlock_file(f):
1199 assert f._lock_file_overlapped_p
1200 handle = msvcrt.get_osfhandle(f.fileno())
1201 if not UnlockFileEx(handle, 0,
1202 whole_low, whole_high, f._lock_file_overlapped_p):
1203 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1204
1205 else:
1206 import fcntl
1207
1208 def _lock_file(f, exclusive):
1209 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1210
1211 def _unlock_file(f):
1212 fcntl.flock(f, fcntl.LOCK_UN)
1213
1214
1215 class locked_file(object):
1216 def __init__(self, filename, mode, encoding=None):
1217 assert mode in ['r', 'a', 'w']
1218 self.f = io.open(filename, mode, encoding=encoding)
1219 self.mode = mode
1220
1221 def __enter__(self):
1222 exclusive = self.mode != 'r'
1223 try:
1224 _lock_file(self.f, exclusive)
1225 except IOError:
1226 self.f.close()
1227 raise
1228 return self
1229
1230 def __exit__(self, etype, value, traceback):
1231 try:
1232 _unlock_file(self.f)
1233 finally:
1234 self.f.close()
1235
1236 def __iter__(self):
1237 return iter(self.f)
1238
1239 def write(self, *args):
1240 return self.f.write(*args)
1241
1242 def read(self, *args):
1243 return self.f.read(*args)
1244
1245
1246 def get_filesystem_encoding():
1247 encoding = sys.getfilesystemencoding()
1248 return encoding if encoding is not None else 'utf-8'
1249
1250
1251 def shell_quote(args):
1252 quoted_args = []
1253 encoding = get_filesystem_encoding()
1254 for a in args:
1255 if isinstance(a, bytes):
1256 # We may get a filename encoded with 'encodeFilename'
1257 a = a.decode(encoding)
1258 quoted_args.append(pipes.quote(a))
1259 return ' '.join(quoted_args)
1260
1261
1262 def smuggle_url(url, data):
1263 """ Pass additional data in a URL for internal use. """
1264
1265 sdata = compat_urllib_parse.urlencode(
1266 {'__youtubedl_smuggle': json.dumps(data)})
1267 return url + '#' + sdata
1268
1269
1270 def unsmuggle_url(smug_url, default=None):
1271 if '#__youtubedl_smuggle' not in smug_url:
1272 return smug_url, default
1273 url, _, sdata = smug_url.rpartition('#')
1274 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1275 data = json.loads(jsond)
1276 return url, data
1277
1278
1279 def format_bytes(bytes):
1280 if bytes is None:
1281 return 'N/A'
1282 if type(bytes) is str:
1283 bytes = float(bytes)
1284 if bytes == 0.0:
1285 exponent = 0
1286 else:
1287 exponent = int(math.log(bytes, 1024.0))
1288 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1289 converted = float(bytes) / float(1024 ** exponent)
1290 return '%.2f%s' % (converted, suffix)
1291
1292
1293 def parse_filesize(s):
1294 if s is None:
1295 return None
1296
1297 # The lower-case forms are of course incorrect and inofficial,
1298 # but we support those too
1299 _UNIT_TABLE = {
1300 'B': 1,
1301 'b': 1,
1302 'KiB': 1024,
1303 'KB': 1000,
1304 'kB': 1024,
1305 'Kb': 1000,
1306 'MiB': 1024 ** 2,
1307 'MB': 1000 ** 2,
1308 'mB': 1024 ** 2,
1309 'Mb': 1000 ** 2,
1310 'GiB': 1024 ** 3,
1311 'GB': 1000 ** 3,
1312 'gB': 1024 ** 3,
1313 'Gb': 1000 ** 3,
1314 'TiB': 1024 ** 4,
1315 'TB': 1000 ** 4,
1316 'tB': 1024 ** 4,
1317 'Tb': 1000 ** 4,
1318 'PiB': 1024 ** 5,
1319 'PB': 1000 ** 5,
1320 'pB': 1024 ** 5,
1321 'Pb': 1000 ** 5,
1322 'EiB': 1024 ** 6,
1323 'EB': 1000 ** 6,
1324 'eB': 1024 ** 6,
1325 'Eb': 1000 ** 6,
1326 'ZiB': 1024 ** 7,
1327 'ZB': 1000 ** 7,
1328 'zB': 1024 ** 7,
1329 'Zb': 1000 ** 7,
1330 'YiB': 1024 ** 8,
1331 'YB': 1000 ** 8,
1332 'yB': 1024 ** 8,
1333 'Yb': 1000 ** 8,
1334 }
1335
1336 units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1337 m = re.match(
1338 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1339 if not m:
1340 return None
1341
1342 num_str = m.group('num').replace(',', '.')
1343 mult = _UNIT_TABLE[m.group('unit')]
1344 return int(float(num_str) * mult)
1345
1346
1347 def month_by_name(name):
1348 """ Return the number of a month by (locale-independently) English name """
1349
1350 try:
1351 return ENGLISH_MONTH_NAMES.index(name) + 1
1352 except ValueError:
1353 return None
1354
1355
1356 def month_by_abbreviation(abbrev):
1357 """ Return the number of a month by (locale-independently) English
1358 abbreviations """
1359
1360 try:
1361 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1362 except ValueError:
1363 return None
1364
1365
1366 def fix_xml_ampersands(xml_str):
1367 """Replace all the '&' by '&amp;' in XML"""
1368 return re.sub(
1369 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1370 '&amp;',
1371 xml_str)
1372
1373
1374 def setproctitle(title):
1375 assert isinstance(title, compat_str)
1376 try:
1377 libc = ctypes.cdll.LoadLibrary("libc.so.6")
1378 except OSError:
1379 return
1380 title_bytes = title.encode('utf-8')
1381 buf = ctypes.create_string_buffer(len(title_bytes))
1382 buf.value = title_bytes
1383 try:
1384 libc.prctl(15, buf, 0, 0, 0)
1385 except AttributeError:
1386 return # Strange libc, just skip this
1387
1388
1389 def remove_start(s, start):
1390 if s.startswith(start):
1391 return s[len(start):]
1392 return s
1393
1394
1395 def remove_end(s, end):
1396 if s.endswith(end):
1397 return s[:-len(end)]
1398 return s
1399
1400
1401 def url_basename(url):
1402 path = compat_urlparse.urlparse(url).path
1403 return path.strip('/').split('/')[-1]
1404
1405
1406 class HEADRequest(compat_urllib_request.Request):
1407 def get_method(self):
1408 return "HEAD"
1409
1410
1411 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1412 if get_attr:
1413 if v is not None:
1414 v = getattr(v, get_attr, None)
1415 if v == '':
1416 v = None
1417 if v is None:
1418 return default
1419 try:
1420 return int(v) * invscale // scale
1421 except ValueError:
1422 return default
1423
1424
1425 def str_or_none(v, default=None):
1426 return default if v is None else compat_str(v)
1427
1428
1429 def str_to_int(int_str):
1430 """ A more relaxed version of int_or_none """
1431 if int_str is None:
1432 return None
1433 int_str = re.sub(r'[,\.\+]', '', int_str)
1434 return int(int_str)
1435
1436
1437 def float_or_none(v, scale=1, invscale=1, default=None):
1438 if v is None:
1439 return default
1440 try:
1441 return float(v) * invscale / scale
1442 except ValueError:
1443 return default
1444
1445
1446 def parse_duration(s):
1447 if not isinstance(s, compat_basestring):
1448 return None
1449
1450 s = s.strip()
1451
1452 m = re.match(
1453 r'''(?ix)(?:P?T)?
1454 (?:
1455 (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1456 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1457
1458 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1459 (?:
1460 (?:
1461 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1462 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1463 )?
1464 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1465 )?
1466 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1467 )$''', s)
1468 if not m:
1469 return None
1470 res = 0
1471 if m.group('only_mins'):
1472 return float_or_none(m.group('only_mins'), invscale=60)
1473 if m.group('only_hours'):
1474 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1475 if m.group('secs'):
1476 res += int(m.group('secs'))
1477 if m.group('mins_reversed'):
1478 res += int(m.group('mins_reversed')) * 60
1479 if m.group('mins'):
1480 res += int(m.group('mins')) * 60
1481 if m.group('hours'):
1482 res += int(m.group('hours')) * 60 * 60
1483 if m.group('hours_reversed'):
1484 res += int(m.group('hours_reversed')) * 60 * 60
1485 if m.group('days'):
1486 res += int(m.group('days')) * 24 * 60 * 60
1487 if m.group('ms'):
1488 res += float(m.group('ms'))
1489 return res
1490
1491
1492 def prepend_extension(filename, ext, expected_real_ext=None):
1493 name, real_ext = os.path.splitext(filename)
1494 return (
1495 '{0}.{1}{2}'.format(name, ext, real_ext)
1496 if not expected_real_ext or real_ext[1:] == expected_real_ext
1497 else '{0}.{1}'.format(filename, ext))
1498
1499
1500 def replace_extension(filename, ext, expected_real_ext=None):
1501 name, real_ext = os.path.splitext(filename)
1502 return '{0}.{1}'.format(
1503 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1504 ext)
1505
1506
1507 def check_executable(exe, args=[]):
1508 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1509 args can be a list of arguments for a short output (like -version) """
1510 try:
1511 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1512 except OSError:
1513 return False
1514 return exe
1515
1516
1517 def get_exe_version(exe, args=['--version'],
1518 version_re=None, unrecognized='present'):
1519 """ Returns the version of the specified executable,
1520 or False if the executable is not present """
1521 try:
1522 out, _ = subprocess.Popen(
1523 [encodeArgument(exe)] + args,
1524 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1525 except OSError:
1526 return False
1527 if isinstance(out, bytes): # Python 2.x
1528 out = out.decode('ascii', 'ignore')
1529 return detect_exe_version(out, version_re, unrecognized)
1530
1531
1532 def detect_exe_version(output, version_re=None, unrecognized='present'):
1533 assert isinstance(output, compat_str)
1534 if version_re is None:
1535 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1536 m = re.search(version_re, output)
1537 if m:
1538 return m.group(1)
1539 else:
1540 return unrecognized
1541
1542
1543 class PagedList(object):
1544 def __len__(self):
1545 # This is only useful for tests
1546 return len(self.getslice())
1547
1548
1549 class OnDemandPagedList(PagedList):
1550 def __init__(self, pagefunc, pagesize):
1551 self._pagefunc = pagefunc
1552 self._pagesize = pagesize
1553
1554 def getslice(self, start=0, end=None):
1555 res = []
1556 for pagenum in itertools.count(start // self._pagesize):
1557 firstid = pagenum * self._pagesize
1558 nextfirstid = pagenum * self._pagesize + self._pagesize
1559 if start >= nextfirstid:
1560 continue
1561
1562 page_results = list(self._pagefunc(pagenum))
1563
1564 startv = (
1565 start % self._pagesize
1566 if firstid <= start < nextfirstid
1567 else 0)
1568
1569 endv = (
1570 ((end - 1) % self._pagesize) + 1
1571 if (end is not None and firstid <= end <= nextfirstid)
1572 else None)
1573
1574 if startv != 0 or endv is not None:
1575 page_results = page_results[startv:endv]
1576 res.extend(page_results)
1577
1578 # A little optimization - if current page is not "full", ie. does
1579 # not contain page_size videos then we can assume that this page
1580 # is the last one - there are no more ids on further pages -
1581 # i.e. no need to query again.
1582 if len(page_results) + startv < self._pagesize:
1583 break
1584
1585 # If we got the whole page, but the next page is not interesting,
1586 # break out early as well
1587 if end == nextfirstid:
1588 break
1589 return res
1590
1591
1592 class InAdvancePagedList(PagedList):
1593 def __init__(self, pagefunc, pagecount, pagesize):
1594 self._pagefunc = pagefunc
1595 self._pagecount = pagecount
1596 self._pagesize = pagesize
1597
1598 def getslice(self, start=0, end=None):
1599 res = []
1600 start_page = start // self._pagesize
1601 end_page = (
1602 self._pagecount if end is None else (end // self._pagesize + 1))
1603 skip_elems = start - start_page * self._pagesize
1604 only_more = None if end is None else end - start
1605 for pagenum in range(start_page, end_page):
1606 page = list(self._pagefunc(pagenum))
1607 if skip_elems:
1608 page = page[skip_elems:]
1609 skip_elems = None
1610 if only_more is not None:
1611 if len(page) < only_more:
1612 only_more -= len(page)
1613 else:
1614 page = page[:only_more]
1615 res.extend(page)
1616 break
1617 res.extend(page)
1618 return res
1619
1620
1621 def uppercase_escape(s):
1622 unicode_escape = codecs.getdecoder('unicode_escape')
1623 return re.sub(
1624 r'\\U[0-9a-fA-F]{8}',
1625 lambda m: unicode_escape(m.group(0))[0],
1626 s)
1627
1628
1629 def lowercase_escape(s):
1630 unicode_escape = codecs.getdecoder('unicode_escape')
1631 return re.sub(
1632 r'\\u[0-9a-fA-F]{4}',
1633 lambda m: unicode_escape(m.group(0))[0],
1634 s)
1635
1636
1637 def escape_rfc3986(s):
1638 """Escape non-ASCII characters as suggested by RFC 3986"""
1639 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1640 s = s.encode('utf-8')
1641 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1642
1643
1644 def escape_url(url):
1645 """Escape URL as suggested by RFC 3986"""
1646 url_parsed = compat_urllib_parse_urlparse(url)
1647 return url_parsed._replace(
1648 path=escape_rfc3986(url_parsed.path),
1649 params=escape_rfc3986(url_parsed.params),
1650 query=escape_rfc3986(url_parsed.query),
1651 fragment=escape_rfc3986(url_parsed.fragment)
1652 ).geturl()
1653
1654 try:
1655 struct.pack('!I', 0)
1656 except TypeError:
1657 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1658 def struct_pack(spec, *args):
1659 if isinstance(spec, compat_str):
1660 spec = spec.encode('ascii')
1661 return struct.pack(spec, *args)
1662
1663 def struct_unpack(spec, *args):
1664 if isinstance(spec, compat_str):
1665 spec = spec.encode('ascii')
1666 return struct.unpack(spec, *args)
1667 else:
1668 struct_pack = struct.pack
1669 struct_unpack = struct.unpack
1670
1671
1672 def read_batch_urls(batch_fd):
1673 def fixup(url):
1674 if not isinstance(url, compat_str):
1675 url = url.decode('utf-8', 'replace')
1676 BOM_UTF8 = '\xef\xbb\xbf'
1677 if url.startswith(BOM_UTF8):
1678 url = url[len(BOM_UTF8):]
1679 url = url.strip()
1680 if url.startswith(('#', ';', ']')):
1681 return False
1682 return url
1683
1684 with contextlib.closing(batch_fd) as fd:
1685 return [url for url in map(fixup, fd) if url]
1686
1687
1688 def urlencode_postdata(*args, **kargs):
1689 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1690
1691
1692 def encode_dict(d, encoding='utf-8'):
1693 def encode(v):
1694 return v.encode(encoding) if isinstance(v, compat_basestring) else v
1695 return dict((encode(k), encode(v)) for k, v in d.items())
1696
1697
1698 US_RATINGS = {
1699 'G': 0,
1700 'PG': 10,
1701 'PG-13': 13,
1702 'R': 16,
1703 'NC': 18,
1704 }
1705
1706
1707 def parse_age_limit(s):
1708 if s is None:
1709 return None
1710 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1711 return int(m.group('age')) if m else US_RATINGS.get(s, None)
1712
1713
1714 def strip_jsonp(code):
1715 return re.sub(
1716 r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1717
1718
1719 def js_to_json(code):
1720 def fix_kv(m):
1721 v = m.group(0)
1722 if v in ('true', 'false', 'null'):
1723 return v
1724 if v.startswith('"'):
1725 v = re.sub(r"\\'", "'", v[1:-1])
1726 elif v.startswith("'"):
1727 v = v[1:-1]
1728 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1729 '\\\\': '\\\\',
1730 "\\'": "'",
1731 '"': '\\"',
1732 }[m.group(0)], v)
1733 return '"%s"' % v
1734
1735 res = re.sub(r'''(?x)
1736 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1737 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1738 [a-zA-Z_][.a-zA-Z_0-9]*
1739 ''', fix_kv, code)
1740 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1741 return res
1742
1743
1744 def qualities(quality_ids):
1745 """ Get a numeric quality value out of a list of possible values """
1746 def q(qid):
1747 try:
1748 return quality_ids.index(qid)
1749 except ValueError:
1750 return -1
1751 return q
1752
1753
1754 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1755
1756
1757 def limit_length(s, length):
1758 """ Add ellipses to overly long strings """
1759 if s is None:
1760 return None
1761 ELLIPSES = '...'
1762 if len(s) > length:
1763 return s[:length - len(ELLIPSES)] + ELLIPSES
1764 return s
1765
1766
1767 def version_tuple(v):
1768 return tuple(int(e) for e in re.split(r'[-.]', v))
1769
1770
1771 def is_outdated_version(version, limit, assume_new=True):
1772 if not version:
1773 return not assume_new
1774 try:
1775 return version_tuple(version) < version_tuple(limit)
1776 except ValueError:
1777 return not assume_new
1778
1779
1780 def ytdl_is_updateable():
1781 """ Returns if youtube-dl can be updated with -U """
1782 from zipimport import zipimporter
1783
1784 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1785
1786
1787 def args_to_str(args):
1788 # Get a short string representation for a subprocess command
1789 return ' '.join(shlex_quote(a) for a in args)
1790
1791
1792 def mimetype2ext(mt):
1793 _, _, res = mt.rpartition('/')
1794
1795 return {
1796 'x-ms-wmv': 'wmv',
1797 'x-mp4-fragmented': 'mp4',
1798 'ttml+xml': 'ttml',
1799 }.get(res, res)
1800
1801
1802 def urlhandle_detect_ext(url_handle):
1803 try:
1804 url_handle.headers
1805 getheader = lambda h: url_handle.headers[h]
1806 except AttributeError: # Python < 3
1807 getheader = url_handle.info().getheader
1808
1809 cd = getheader('Content-Disposition')
1810 if cd:
1811 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1812 if m:
1813 e = determine_ext(m.group('filename'), default_ext=None)
1814 if e:
1815 return e
1816
1817 return mimetype2ext(getheader('Content-Type'))
1818
1819
1820 def encode_data_uri(data, mime_type):
1821 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
1822
1823
1824 def age_restricted(content_limit, age_limit):
1825 """ Returns True iff the content should be blocked """
1826
1827 if age_limit is None: # No limit set
1828 return False
1829 if content_limit is None:
1830 return False # Content available for everyone
1831 return age_limit < content_limit
1832
1833
1834 def is_html(first_bytes):
1835 """ Detect whether a file contains HTML by examining its first bytes. """
1836
1837 BOMS = [
1838 (b'\xef\xbb\xbf', 'utf-8'),
1839 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1840 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1841 (b'\xff\xfe', 'utf-16-le'),
1842 (b'\xfe\xff', 'utf-16-be'),
1843 ]
1844 for bom, enc in BOMS:
1845 if first_bytes.startswith(bom):
1846 s = first_bytes[len(bom):].decode(enc, 'replace')
1847 break
1848 else:
1849 s = first_bytes.decode('utf-8', 'replace')
1850
1851 return re.match(r'^\s*<', s)
1852
1853
1854 def determine_protocol(info_dict):
1855 protocol = info_dict.get('protocol')
1856 if protocol is not None:
1857 return protocol
1858
1859 url = info_dict['url']
1860 if url.startswith('rtmp'):
1861 return 'rtmp'
1862 elif url.startswith('mms'):
1863 return 'mms'
1864 elif url.startswith('rtsp'):
1865 return 'rtsp'
1866
1867 ext = determine_ext(url)
1868 if ext == 'm3u8':
1869 return 'm3u8'
1870 elif ext == 'f4m':
1871 return 'f4m'
1872
1873 return compat_urllib_parse_urlparse(url).scheme
1874
1875
1876 def render_table(header_row, data):
1877 """ Render a list of rows, each as a list of values """
1878 table = [header_row] + data
1879 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1880 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1881 return '\n'.join(format_str % tuple(row) for row in table)
1882
1883
1884 def _match_one(filter_part, dct):
1885 COMPARISON_OPERATORS = {
1886 '<': operator.lt,
1887 '<=': operator.le,
1888 '>': operator.gt,
1889 '>=': operator.ge,
1890 '=': operator.eq,
1891 '!=': operator.ne,
1892 }
1893 operator_rex = re.compile(r'''(?x)\s*
1894 (?P<key>[a-z_]+)
1895 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1896 (?:
1897 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1898 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1899 )
1900 \s*$
1901 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1902 m = operator_rex.search(filter_part)
1903 if m:
1904 op = COMPARISON_OPERATORS[m.group('op')]
1905 if m.group('strval') is not None:
1906 if m.group('op') not in ('=', '!='):
1907 raise ValueError(
1908 'Operator %s does not support string values!' % m.group('op'))
1909 comparison_value = m.group('strval')
1910 else:
1911 try:
1912 comparison_value = int(m.group('intval'))
1913 except ValueError:
1914 comparison_value = parse_filesize(m.group('intval'))
1915 if comparison_value is None:
1916 comparison_value = parse_filesize(m.group('intval') + 'B')
1917 if comparison_value is None:
1918 raise ValueError(
1919 'Invalid integer value %r in filter part %r' % (
1920 m.group('intval'), filter_part))
1921 actual_value = dct.get(m.group('key'))
1922 if actual_value is None:
1923 return m.group('none_inclusive')
1924 return op(actual_value, comparison_value)
1925
1926 UNARY_OPERATORS = {
1927 '': lambda v: v is not None,
1928 '!': lambda v: v is None,
1929 }
1930 operator_rex = re.compile(r'''(?x)\s*
1931 (?P<op>%s)\s*(?P<key>[a-z_]+)
1932 \s*$
1933 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1934 m = operator_rex.search(filter_part)
1935 if m:
1936 op = UNARY_OPERATORS[m.group('op')]
1937 actual_value = dct.get(m.group('key'))
1938 return op(actual_value)
1939
1940 raise ValueError('Invalid filter part %r' % filter_part)
1941
1942
1943 def match_str(filter_str, dct):
1944 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1945
1946 return all(
1947 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1948
1949
1950 def match_filter_func(filter_str):
1951 def _match_func(info_dict):
1952 if match_str(filter_str, info_dict):
1953 return None
1954 else:
1955 video_title = info_dict.get('title', info_dict.get('id', 'video'))
1956 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1957 return _match_func
1958
1959
1960 def parse_dfxp_time_expr(time_expr):
1961 if not time_expr:
1962 return 0.0
1963
1964 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
1965 if mobj:
1966 return float(mobj.group('time_offset'))
1967
1968 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
1969 if mobj:
1970 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
1971
1972
1973 def srt_subtitles_timecode(seconds):
1974 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
1975
1976
1977 def dfxp2srt(dfxp_data):
1978 _x = functools.partial(xpath_with_ns, ns_map={
1979 'ttml': 'http://www.w3.org/ns/ttml',
1980 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
1981 })
1982
1983 def parse_node(node):
1984 str_or_empty = functools.partial(str_or_none, default='')
1985
1986 out = str_or_empty(node.text)
1987
1988 for child in node:
1989 if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
1990 out += '\n' + str_or_empty(child.tail)
1991 elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
1992 out += str_or_empty(parse_node(child))
1993 else:
1994 out += str_or_empty(xml.etree.ElementTree.tostring(child))
1995
1996 return out
1997
1998 dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
1999 out = []
2000 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
2001
2002 if not paras:
2003 raise ValueError('Invalid dfxp/TTML subtitle')
2004
2005 for para, index in zip(paras, itertools.count(1)):
2006 begin_time = parse_dfxp_time_expr(para.attrib['begin'])
2007 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2008 if not end_time:
2009 end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
2010 out.append('%d\n%s --> %s\n%s\n\n' % (
2011 index,
2012 srt_subtitles_timecode(begin_time),
2013 srt_subtitles_timecode(end_time),
2014 parse_node(para)))
2015
2016 return ''.join(out)
2017
2018
2019 def cli_option(params, command_option, param):
2020 param = params.get(param)
2021 return [command_option, param] if param is not None else []
2022
2023
2024 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2025 param = params.get(param)
2026 assert isinstance(param, bool)
2027 if separator:
2028 return [command_option + separator + (true_value if param else false_value)]
2029 return [command_option, true_value if param else false_value]
2030
2031
2032 def cli_valueless_option(params, command_option, param, expected_value=True):
2033 param = params.get(param)
2034 return [command_option] if param == expected_value else []
2035
2036
2037 def cli_configuration_args(params, param, default=[]):
2038 ex_args = params.get(param)
2039 if ex_args is None:
2040 return default
2041 assert isinstance(ex_args, list)
2042 return ex_args
2043
2044
2045 class ISO639Utils(object):
2046 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2047 _lang_map = {
2048 'aa': 'aar',
2049 'ab': 'abk',
2050 'ae': 'ave',
2051 'af': 'afr',
2052 'ak': 'aka',
2053 'am': 'amh',
2054 'an': 'arg',
2055 'ar': 'ara',
2056 'as': 'asm',
2057 'av': 'ava',
2058 'ay': 'aym',
2059 'az': 'aze',
2060 'ba': 'bak',
2061 'be': 'bel',
2062 'bg': 'bul',
2063 'bh': 'bih',
2064 'bi': 'bis',
2065 'bm': 'bam',
2066 'bn': 'ben',
2067 'bo': 'bod',
2068 'br': 'bre',
2069 'bs': 'bos',
2070 'ca': 'cat',
2071 'ce': 'che',
2072 'ch': 'cha',
2073 'co': 'cos',
2074 'cr': 'cre',
2075 'cs': 'ces',
2076 'cu': 'chu',
2077 'cv': 'chv',
2078 'cy': 'cym',
2079 'da': 'dan',
2080 'de': 'deu',
2081 'dv': 'div',
2082 'dz': 'dzo',
2083 'ee': 'ewe',
2084 'el': 'ell',
2085 'en': 'eng',
2086 'eo': 'epo',
2087 'es': 'spa',
2088 'et': 'est',
2089 'eu': 'eus',
2090 'fa': 'fas',
2091 'ff': 'ful',
2092 'fi': 'fin',
2093 'fj': 'fij',
2094 'fo': 'fao',
2095 'fr': 'fra',
2096 'fy': 'fry',
2097 'ga': 'gle',
2098 'gd': 'gla',
2099 'gl': 'glg',
2100 'gn': 'grn',
2101 'gu': 'guj',
2102 'gv': 'glv',
2103 'ha': 'hau',
2104 'he': 'heb',
2105 'hi': 'hin',
2106 'ho': 'hmo',
2107 'hr': 'hrv',
2108 'ht': 'hat',
2109 'hu': 'hun',
2110 'hy': 'hye',
2111 'hz': 'her',
2112 'ia': 'ina',
2113 'id': 'ind',
2114 'ie': 'ile',
2115 'ig': 'ibo',
2116 'ii': 'iii',
2117 'ik': 'ipk',
2118 'io': 'ido',
2119 'is': 'isl',
2120 'it': 'ita',
2121 'iu': 'iku',
2122 'ja': 'jpn',
2123 'jv': 'jav',
2124 'ka': 'kat',
2125 'kg': 'kon',
2126 'ki': 'kik',
2127 'kj': 'kua',
2128 'kk': 'kaz',
2129 'kl': 'kal',
2130 'km': 'khm',
2131 'kn': 'kan',
2132 'ko': 'kor',
2133 'kr': 'kau',
2134 'ks': 'kas',
2135 'ku': 'kur',
2136 'kv': 'kom',
2137 'kw': 'cor',
2138 'ky': 'kir',
2139 'la': 'lat',
2140 'lb': 'ltz',
2141 'lg': 'lug',
2142 'li': 'lim',
2143 'ln': 'lin',
2144 'lo': 'lao',
2145 'lt': 'lit',
2146 'lu': 'lub',
2147 'lv': 'lav',
2148 'mg': 'mlg',
2149 'mh': 'mah',
2150 'mi': 'mri',
2151 'mk': 'mkd',
2152 'ml': 'mal',
2153 'mn': 'mon',
2154 'mr': 'mar',
2155 'ms': 'msa',
2156 'mt': 'mlt',
2157 'my': 'mya',
2158 'na': 'nau',
2159 'nb': 'nob',
2160 'nd': 'nde',
2161 'ne': 'nep',
2162 'ng': 'ndo',
2163 'nl': 'nld',
2164 'nn': 'nno',
2165 'no': 'nor',
2166 'nr': 'nbl',
2167 'nv': 'nav',
2168 'ny': 'nya',
2169 'oc': 'oci',
2170 'oj': 'oji',
2171 'om': 'orm',
2172 'or': 'ori',
2173 'os': 'oss',
2174 'pa': 'pan',
2175 'pi': 'pli',
2176 'pl': 'pol',
2177 'ps': 'pus',
2178 'pt': 'por',
2179 'qu': 'que',
2180 'rm': 'roh',
2181 'rn': 'run',
2182 'ro': 'ron',
2183 'ru': 'rus',
2184 'rw': 'kin',
2185 'sa': 'san',
2186 'sc': 'srd',
2187 'sd': 'snd',
2188 'se': 'sme',
2189 'sg': 'sag',
2190 'si': 'sin',
2191 'sk': 'slk',
2192 'sl': 'slv',
2193 'sm': 'smo',
2194 'sn': 'sna',
2195 'so': 'som',
2196 'sq': 'sqi',
2197 'sr': 'srp',
2198 'ss': 'ssw',
2199 'st': 'sot',
2200 'su': 'sun',
2201 'sv': 'swe',
2202 'sw': 'swa',
2203 'ta': 'tam',
2204 'te': 'tel',
2205 'tg': 'tgk',
2206 'th': 'tha',
2207 'ti': 'tir',
2208 'tk': 'tuk',
2209 'tl': 'tgl',
2210 'tn': 'tsn',
2211 'to': 'ton',
2212 'tr': 'tur',
2213 'ts': 'tso',
2214 'tt': 'tat',
2215 'tw': 'twi',
2216 'ty': 'tah',
2217 'ug': 'uig',
2218 'uk': 'ukr',
2219 'ur': 'urd',
2220 'uz': 'uzb',
2221 've': 'ven',
2222 'vi': 'vie',
2223 'vo': 'vol',
2224 'wa': 'wln',
2225 'wo': 'wol',
2226 'xh': 'xho',
2227 'yi': 'yid',
2228 'yo': 'yor',
2229 'za': 'zha',
2230 'zh': 'zho',
2231 'zu': 'zul',
2232 }
2233
2234 @classmethod
2235 def short2long(cls, code):
2236 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2237 return cls._lang_map.get(code[:2])
2238
2239 @classmethod
2240 def long2short(cls, code):
2241 """Convert language code from ISO 639-2/T to ISO 639-1"""
2242 for short_name, long_name in cls._lang_map.items():
2243 if long_name == code:
2244 return short_name
2245
2246
2247 class ISO3166Utils(object):
2248 # From http://data.okfn.org/data/core/country-list
2249 _country_map = {
2250 'AF': 'Afghanistan',
2251 'AX': 'Åland Islands',
2252 'AL': 'Albania',
2253 'DZ': 'Algeria',
2254 'AS': 'American Samoa',
2255 'AD': 'Andorra',
2256 'AO': 'Angola',
2257 'AI': 'Anguilla',
2258 'AQ': 'Antarctica',
2259 'AG': 'Antigua and Barbuda',
2260 'AR': 'Argentina',
2261 'AM': 'Armenia',
2262 'AW': 'Aruba',
2263 'AU': 'Australia',
2264 'AT': 'Austria',
2265 'AZ': 'Azerbaijan',
2266 'BS': 'Bahamas',
2267 'BH': 'Bahrain',
2268 'BD': 'Bangladesh',
2269 'BB': 'Barbados',
2270 'BY': 'Belarus',
2271 'BE': 'Belgium',
2272 'BZ': 'Belize',
2273 'BJ': 'Benin',
2274 'BM': 'Bermuda',
2275 'BT': 'Bhutan',
2276 'BO': 'Bolivia, Plurinational State of',
2277 'BQ': 'Bonaire, Sint Eustatius and Saba',
2278 'BA': 'Bosnia and Herzegovina',
2279 'BW': 'Botswana',
2280 'BV': 'Bouvet Island',
2281 'BR': 'Brazil',
2282 'IO': 'British Indian Ocean Territory',
2283 'BN': 'Brunei Darussalam',
2284 'BG': 'Bulgaria',
2285 'BF': 'Burkina Faso',
2286 'BI': 'Burundi',
2287 'KH': 'Cambodia',
2288 'CM': 'Cameroon',
2289 'CA': 'Canada',
2290 'CV': 'Cape Verde',
2291 'KY': 'Cayman Islands',
2292 'CF': 'Central African Republic',
2293 'TD': 'Chad',
2294 'CL': 'Chile',
2295 'CN': 'China',
2296 'CX': 'Christmas Island',
2297 'CC': 'Cocos (Keeling) Islands',
2298 'CO': 'Colombia',
2299 'KM': 'Comoros',
2300 'CG': 'Congo',
2301 'CD': 'Congo, the Democratic Republic of the',
2302 'CK': 'Cook Islands',
2303 'CR': 'Costa Rica',
2304 'CI': 'Côte d\'Ivoire',
2305 'HR': 'Croatia',
2306 'CU': 'Cuba',
2307 'CW': 'Curaçao',
2308 'CY': 'Cyprus',
2309 'CZ': 'Czech Republic',
2310 'DK': 'Denmark',
2311 'DJ': 'Djibouti',
2312 'DM': 'Dominica',
2313 'DO': 'Dominican Republic',
2314 'EC': 'Ecuador',
2315 'EG': 'Egypt',
2316 'SV': 'El Salvador',
2317 'GQ': 'Equatorial Guinea',
2318 'ER': 'Eritrea',
2319 'EE': 'Estonia',
2320 'ET': 'Ethiopia',
2321 'FK': 'Falkland Islands (Malvinas)',
2322 'FO': 'Faroe Islands',
2323 'FJ': 'Fiji',
2324 'FI': 'Finland',
2325 'FR': 'France',
2326 'GF': 'French Guiana',
2327 'PF': 'French Polynesia',
2328 'TF': 'French Southern Territories',
2329 'GA': 'Gabon',
2330 'GM': 'Gambia',
2331 'GE': 'Georgia',
2332 'DE': 'Germany',
2333 'GH': 'Ghana',
2334 'GI': 'Gibraltar',
2335 'GR': 'Greece',
2336 'GL': 'Greenland',
2337 'GD': 'Grenada',
2338 'GP': 'Guadeloupe',
2339 'GU': 'Guam',
2340 'GT': 'Guatemala',
2341 'GG': 'Guernsey',
2342 'GN': 'Guinea',
2343 'GW': 'Guinea-Bissau',
2344 'GY': 'Guyana',
2345 'HT': 'Haiti',
2346 'HM': 'Heard Island and McDonald Islands',
2347 'VA': 'Holy See (Vatican City State)',
2348 'HN': 'Honduras',
2349 'HK': 'Hong Kong',
2350 'HU': 'Hungary',
2351 'IS': 'Iceland',
2352 'IN': 'India',
2353 'ID': 'Indonesia',
2354 'IR': 'Iran, Islamic Republic of',
2355 'IQ': 'Iraq',
2356 'IE': 'Ireland',
2357 'IM': 'Isle of Man',
2358 'IL': 'Israel',
2359 'IT': 'Italy',
2360 'JM': 'Jamaica',
2361 'JP': 'Japan',
2362 'JE': 'Jersey',
2363 'JO': 'Jordan',
2364 'KZ': 'Kazakhstan',
2365 'KE': 'Kenya',
2366 'KI': 'Kiribati',
2367 'KP': 'Korea, Democratic People\'s Republic of',
2368 'KR': 'Korea, Republic of',
2369 'KW': 'Kuwait',
2370 'KG': 'Kyrgyzstan',
2371 'LA': 'Lao People\'s Democratic Republic',
2372 'LV': 'Latvia',
2373 'LB': 'Lebanon',
2374 'LS': 'Lesotho',
2375 'LR': 'Liberia',
2376 'LY': 'Libya',
2377 'LI': 'Liechtenstein',
2378 'LT': 'Lithuania',
2379 'LU': 'Luxembourg',
2380 'MO': 'Macao',
2381 'MK': 'Macedonia, the Former Yugoslav Republic of',
2382 'MG': 'Madagascar',
2383 'MW': 'Malawi',
2384 'MY': 'Malaysia',
2385 'MV': 'Maldives',
2386 'ML': 'Mali',
2387 'MT': 'Malta',
2388 'MH': 'Marshall Islands',
2389 'MQ': 'Martinique',
2390 'MR': 'Mauritania',
2391 'MU': 'Mauritius',
2392 'YT': 'Mayotte',
2393 'MX': 'Mexico',
2394 'FM': 'Micronesia, Federated States of',
2395 'MD': 'Moldova, Republic of',
2396 'MC': 'Monaco',
2397 'MN': 'Mongolia',
2398 'ME': 'Montenegro',
2399 'MS': 'Montserrat',
2400 'MA': 'Morocco',
2401 'MZ': 'Mozambique',
2402 'MM': 'Myanmar',
2403 'NA': 'Namibia',
2404 'NR': 'Nauru',
2405 'NP': 'Nepal',
2406 'NL': 'Netherlands',
2407 'NC': 'New Caledonia',
2408 'NZ': 'New Zealand',
2409 'NI': 'Nicaragua',
2410 'NE': 'Niger',
2411 'NG': 'Nigeria',
2412 'NU': 'Niue',
2413 'NF': 'Norfolk Island',
2414 'MP': 'Northern Mariana Islands',
2415 'NO': 'Norway',
2416 'OM': 'Oman',
2417 'PK': 'Pakistan',
2418 'PW': 'Palau',
2419 'PS': 'Palestine, State of',
2420 'PA': 'Panama',
2421 'PG': 'Papua New Guinea',
2422 'PY': 'Paraguay',
2423 'PE': 'Peru',
2424 'PH': 'Philippines',
2425 'PN': 'Pitcairn',
2426 'PL': 'Poland',
2427 'PT': 'Portugal',
2428 'PR': 'Puerto Rico',
2429 'QA': 'Qatar',
2430 'RE': 'Réunion',
2431 'RO': 'Romania',
2432 'RU': 'Russian Federation',
2433 'RW': 'Rwanda',
2434 'BL': 'Saint Barthélemy',
2435 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2436 'KN': 'Saint Kitts and Nevis',
2437 'LC': 'Saint Lucia',
2438 'MF': 'Saint Martin (French part)',
2439 'PM': 'Saint Pierre and Miquelon',
2440 'VC': 'Saint Vincent and the Grenadines',
2441 'WS': 'Samoa',
2442 'SM': 'San Marino',
2443 'ST': 'Sao Tome and Principe',
2444 'SA': 'Saudi Arabia',
2445 'SN': 'Senegal',
2446 'RS': 'Serbia',
2447 'SC': 'Seychelles',
2448 'SL': 'Sierra Leone',
2449 'SG': 'Singapore',
2450 'SX': 'Sint Maarten (Dutch part)',
2451 'SK': 'Slovakia',
2452 'SI': 'Slovenia',
2453 'SB': 'Solomon Islands',
2454 'SO': 'Somalia',
2455 'ZA': 'South Africa',
2456 'GS': 'South Georgia and the South Sandwich Islands',
2457 'SS': 'South Sudan',
2458 'ES': 'Spain',
2459 'LK': 'Sri Lanka',
2460 'SD': 'Sudan',
2461 'SR': 'Suriname',
2462 'SJ': 'Svalbard and Jan Mayen',
2463 'SZ': 'Swaziland',
2464 'SE': 'Sweden',
2465 'CH': 'Switzerland',
2466 'SY': 'Syrian Arab Republic',
2467 'TW': 'Taiwan, Province of China',
2468 'TJ': 'Tajikistan',
2469 'TZ': 'Tanzania, United Republic of',
2470 'TH': 'Thailand',
2471 'TL': 'Timor-Leste',
2472 'TG': 'Togo',
2473 'TK': 'Tokelau',
2474 'TO': 'Tonga',
2475 'TT': 'Trinidad and Tobago',
2476 'TN': 'Tunisia',
2477 'TR': 'Turkey',
2478 'TM': 'Turkmenistan',
2479 'TC': 'Turks and Caicos Islands',
2480 'TV': 'Tuvalu',
2481 'UG': 'Uganda',
2482 'UA': 'Ukraine',
2483 'AE': 'United Arab Emirates',
2484 'GB': 'United Kingdom',
2485 'US': 'United States',
2486 'UM': 'United States Minor Outlying Islands',
2487 'UY': 'Uruguay',
2488 'UZ': 'Uzbekistan',
2489 'VU': 'Vanuatu',
2490 'VE': 'Venezuela, Bolivarian Republic of',
2491 'VN': 'Viet Nam',
2492 'VG': 'Virgin Islands, British',
2493 'VI': 'Virgin Islands, U.S.',
2494 'WF': 'Wallis and Futuna',
2495 'EH': 'Western Sahara',
2496 'YE': 'Yemen',
2497 'ZM': 'Zambia',
2498 'ZW': 'Zimbabwe',
2499 }
2500
2501 @classmethod
2502 def short2full(cls, code):
2503 """Convert an ISO 3166-2 country code to the corresponding full name"""
2504 return cls._country_map.get(code.upper())
2505
2506
2507 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2508 def __init__(self, proxies=None):
2509 # Set default handlers
2510 for type in ('http', 'https'):
2511 setattr(self, '%s_open' % type,
2512 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2513 meth(r, proxy, type))
2514 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2515
2516 def proxy_open(self, req, proxy, type):
2517 req_proxy = req.headers.get('Ytdl-request-proxy')
2518 if req_proxy is not None:
2519 proxy = req_proxy
2520 del req.headers['Ytdl-request-proxy']
2521
2522 if proxy == '__noproxy__':
2523 return None # No Proxy
2524 return compat_urllib_request.ProxyHandler.proxy_open(
2525 self, req, proxy, type)