Register new version in the changelog.

[youtubedl] / youtube_dl / compat.py

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0c57c7aebf6bd45a8f9edfc7e1d585f1bbf2f23a..a3e85264acda8dbefe0883ea50c901302d01d29f 100644 (file)
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,15 +1,20 @@
 from __future__ import unicode_literals
 
 from __future__ import unicode_literals
 

+import binascii

 import collections
 import collections

+import email

 import getpass
 import getpass

+import io

 import optparse
 import os
 import re
 import optparse
 import os
 import re

+import shlex

 import shutil
 import socket
 import subprocess
 import sys
 import itertools
 import shutil
 import socket
 import subprocess
 import sys
 import itertools

+import xml.etree.ElementTree

 
 
 try:
 
 
 try:


@@ -37,11 +42,21 @@ try:
 except ImportError:  # Python 2
     import urlparse as compat_urlparse
 
 except ImportError:  # Python 2
     import urlparse as compat_urlparse
 

+try:
+    import urllib.response as compat_urllib_response
+except ImportError:  # Python 2
+    import urllib as compat_urllib_response
+

 try:
     import http.cookiejar as compat_cookiejar
 except ImportError:  # Python 2
     import cookielib as compat_cookiejar
 
 try:
     import http.cookiejar as compat_cookiejar
 except ImportError:  # Python 2
     import cookielib as compat_cookiejar
 

+try:
+    import http.cookies as compat_cookies
+except ImportError:  # Python 2
+    import Cookie as compat_cookies
+

 try:
     import html.entities as compat_html_entities
 except ImportError:  # Python 2
 try:
     import html.entities as compat_html_entities
 except ImportError:  # Python 2


@@ -74,6 +89,11 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
 except ImportError:
     import BaseHTTPServer as compat_http_server
 

+try:
+    compat_str = unicode  # Python 2
+except NameError:
+    compat_str = str
+

 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote


@@ -94,7 +114,7 @@ except ImportError:  # Python 2
             # Is it a string-like object?
             string.split
             return b''
             # Is it a string-like object?
             string.split
             return b''

-        if isinstance(string, unicode):
+        if isinstance(string, compat_str):

             string = string.encode('utf-8')
         bits = string.split(b'%')
         if len(bits) == 1:
             string = string.encode('utf-8')
         bits = string.split(b'%')
         if len(bits) == 1:


@@ -145,9 +165,38 @@ except ImportError:  # Python 2
         return compat_urllib_parse_unquote(string, encoding, errors)
 
 try:
         return compat_urllib_parse_unquote(string, encoding, errors)
 
 try:

-    compat_str = unicode  # Python 2
-except NameError:
-    compat_str = str
+    from urllib.request import DataHandler as compat_urllib_request_DataHandler
+except ImportError:  # Python < 3.4
+    # Ported from CPython 98774:1733b3bd46db, Lib/urllib/request.py
+    class compat_urllib_request_DataHandler(compat_urllib_request.BaseHandler):
+        def data_open(self, req):
+            # data URLs as specified in RFC 2397.
+            #
+            # ignores POSTed data
+            #
+            # syntax:
+            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
+            # mediatype := [ type "/" subtype ] *( ";" parameter )
+            # data      := *urlchar
+            # parameter := attribute "=" value
+            url = req.get_full_url()
+
+            scheme, data = url.split(":", 1)
+            mediatype, data = data.split(",", 1)
+
+            # even base64 encoded data URLs might be quoted so unquote in any case:
+            data = compat_urllib_parse_unquote_to_bytes(data)
+            if mediatype.endswith(";base64"):
+                data = binascii.a2b_base64(data)
+                mediatype = mediatype[:-7]
+
+            if not mediatype:
+                mediatype = "text/plain;charset=US-ASCII"
+
+            headers = email.message_from_string(
+                "Content-type: %s\nContent-length: %d\n" % (mediatype, len(data)))
+
+            return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)

 
 try:
     compat_basestring = basestring  # Python 2
 
 try:
     compat_basestring = basestring  # Python 2


@@ -164,6 +213,43 @@ try:
 except ImportError:  # Python 2.6
     from xml.parsers.expat import ExpatError as compat_xml_parse_error
 
 except ImportError:  # Python 2.6
     from xml.parsers.expat import ExpatError as compat_xml_parse_error
 

+if sys.version_info[0] >= 3:
+    compat_etree_fromstring = xml.etree.ElementTree.fromstring
+else:
+    # python 2.x tries to encode unicode strings with ascii (see the
+    # XMLParser._fixtext method)
+    etree = xml.etree.ElementTree
+
+    try:
+        _etree_iter = etree.Element.iter
+    except AttributeError:  # Python <=2.6
+        def _etree_iter(root):
+            for el in root.findall('*'):
+                yield el
+                for sub in _etree_iter(el):
+                    yield sub
+
+    # on 2.6 XML doesn't have a parser argument, function copied from CPython
+    # 2.7 source
+    def _XML(text, parser=None):
+        if not parser:
+            parser = etree.XMLParser(target=etree.TreeBuilder())
+        parser.feed(text)
+        return parser.close()
+
+    def _element_factory(*args, **kwargs):
+        el = etree.Element(*args, **kwargs)
+        for k, v in el.items():
+            if isinstance(v, bytes):
+                el.set(k, v.decode('utf-8'))
+        return el
+
+    def compat_etree_fromstring(text):
+        doc = _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+        for el in _etree_iter(doc):
+            if el.text is not None and isinstance(el.text, bytes):
+                el.text = el.text.decode('utf-8')
+        return doc

 
 try:
     from urllib.parse import parse_qs as compat_parse_qs
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs


@@ -222,6 +308,17 @@ except ImportError:  # Python < 3.3
             return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
             return "'" + s.replace("'", "'\"'\"'") + "'"
 
 

+if sys.version_info >= (2, 7, 3):
+    compat_shlex_split = shlex.split
+else:
+    # Working around shlex issue with unicode strings on some python 2
+    # versions (see http://bugs.python.org/issue1548891)
+    def compat_shlex_split(s, comments=False, posix=True):
+        if isinstance(s, compat_str):
+            s = s.encode('utf-8')
+        return shlex.split(s, comments, posix)
+
+

 def compat_ord(c):
     if type(c) is int:
         return c
 def compat_ord(c):
     if type(c) is int:
         return c


@@ -399,26 +496,32 @@ if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
 else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
 else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 

-    def compat_get_terminal_size():
-        columns = compat_getenv('COLUMNS', None)
+    def compat_get_terminal_size(fallback=(80, 24)):
+        columns = compat_getenv('COLUMNS')

         if columns:
             columns = int(columns)
         else:
             columns = None
         if columns:
             columns = int(columns)
         else:
             columns = None

-        lines = compat_getenv('LINES', None)
+        lines = compat_getenv('LINES')

         if lines:
             lines = int(lines)
         else:
             lines = None
 
         if lines:
             lines = int(lines)
         else:
             lines = None
 

-        try:
-            sp = subprocess.Popen(
-                ['stty', 'size'],
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            out, err = sp.communicate()
-            lines, columns = map(int, out.split())
-        except Exception:
-            pass
+        if columns is None or lines is None or columns <= 0 or lines <= 0:
+            try:
+                sp = subprocess.Popen(
+                    ['stty', 'size'],
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                out, err = sp.communicate()
+                _lines, _columns = map(int, out.split())
+            except Exception:
+                _columns, _lines = _terminal_size(*fallback)
+
+            if columns is None or columns <= 0:
+                columns = _columns
+            if lines is None or lines <= 0:
+                lines = _lines

         return _terminal_size(columns, lines)
 
 try:
         return _terminal_size(columns, lines)
 
 try:


@@ -431,11 +534,18 @@ except TypeError:  # Python 2.6
             yield n
             n += step
 
             yield n
             n += step
 

+if sys.version_info >= (3, 0):
+    from tokenize import tokenize as compat_tokenize_tokenize
+else:
+    from tokenize import generate_tokens as compat_tokenize_tokenize
+

 __all__ = [
     'compat_HTTPError',
     'compat_basestring',
     'compat_chr',
     'compat_cookiejar',
 __all__ = [
     'compat_HTTPError',
     'compat_basestring',
     'compat_chr',
     'compat_cookiejar',

+    'compat_cookies',
+    'compat_etree_fromstring',

     'compat_expanduser',
     'compat_get_terminal_size',
     'compat_getenv',
     'compat_expanduser',
     'compat_get_terminal_size',
     'compat_getenv',


@@ -448,9 +558,11 @@ __all__ = [
     'compat_ord',
     'compat_parse_qs',
     'compat_print',
     'compat_ord',
     'compat_parse_qs',
     'compat_print',

+    'compat_shlex_split',

     'compat_socket_create_connection',
     'compat_str',
     'compat_subprocess_get_DEVNULL',
     'compat_socket_create_connection',
     'compat_str',
     'compat_subprocess_get_DEVNULL',

+    'compat_tokenize_tokenize',

     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_urllib_parse_unquote',
     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_urllib_parse_unquote',


@@ -458,6 +570,8 @@ __all__ = [
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',
     'compat_urllib_parse_unquote_to_bytes',
     'compat_urllib_parse_urlparse',
     'compat_urllib_request',

+    'compat_urllib_request_DataHandler',
+    'compat_urllib_response',

     'compat_urlparse',
     'compat_urlretrieve',
     'compat_xml_parse_error',
     'compat_urlparse',
     'compat_urlretrieve',
     'compat_xml_parse_error',