Revert "debian/watch: Add back pgpsigurlmangle."

[youtubedl] / youtube_dl / utils.py

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6978a10e4cfa9b89874f5ec1da15c30a3e6d7e99..562031fe110a9a6962c6e5efed2bb311f3059979 100644 (file)
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -14,8 +14,8 @@ import email.utils
 import errno
 import functools
 import gzip
 import errno
 import functools
 import gzip

-import itertools

 import io
 import io

+import itertools

 import json
 import locale
 import math
 import json
 import locale
 import math


@@ -24,9 +24,8 @@ import os
 import pipes
 import platform
 import re
 import pipes
 import platform
 import re

-import ssl

 import socket
 import socket

-import struct
+import ssl

 import subprocess
 import sys
 import tempfile
 import subprocess
 import sys
 import tempfile


@@ -35,29 +34,49 @@ import xml.etree.ElementTree
 import zlib
 
 from .compat import (
 import zlib
 
 from .compat import (

+    compat_HTMLParser,

     compat_basestring,
     compat_chr,
     compat_etree_fromstring,
     compat_html_entities,
     compat_basestring,
     compat_chr,
     compat_etree_fromstring,
     compat_html_entities,

+    compat_html_entities_html5,

     compat_http_client,
     compat_kwargs,
     compat_parse_qs,
     compat_http_client,
     compat_kwargs,
     compat_parse_qs,

+    compat_shlex_quote,

     compat_socket_create_connection,
     compat_str,
     compat_socket_create_connection,
     compat_str,

+    compat_struct_pack,

     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_error,
     compat_urllib_parse,

+    compat_urllib_parse_urlencode,

     compat_urllib_parse_urlparse,
     compat_urllib_parse_urlparse,

+    compat_urllib_parse_unquote_plus,

     compat_urllib_request,
     compat_urlparse,
     compat_urllib_request,
     compat_urlparse,

-    shlex_quote,
+    compat_xpath,

 )
 
 )
 

+from .socks import (
+    ProxyType,
+    sockssocket,
+)
+
+
+def register_socks_protocols():
+    # "Register" SOCKS protocols
+    # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
+    # URLs with protocols not in urlparse.uses_netloc are not handled correctly
+    for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
+        if scheme not in compat_urlparse.uses_netloc:
+            compat_urlparse.uses_netloc.append(scheme)
+

 
 # This is not clearly defined otherwise
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {
 
 # This is not clearly defined otherwise
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {

-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/44.0 (Chrome)',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/47.0 (Chrome)',

     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',


@@ -86,6 +105,11 @@ KNOWN_EXTENSIONS = (
     'wav',
     'f4f', 'f4m', 'm3u8', 'smil')
 
     'wav',
     'f4f', 'f4m', 'm3u8', 'smil')
 

+# needed for sanitizing filenames in restricted mode
+ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
+                        itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUYP', ['ss'],
+                                        'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuypy')))
+

 
 def preferredencoding():
     """Get preferred encoding.
 
 def preferredencoding():
     """Get preferred encoding.


@@ -160,18 +184,11 @@ if sys.version_info >= (2, 7):
     def find_xpath_attr(node, xpath, key, val=None):
         """ Find the xpath xpath[@key=val] """
         assert re.match(r'^[a-zA-Z_-]+$', key)
     def find_xpath_attr(node, xpath, key, val=None):
         """ Find the xpath xpath[@key=val] """
         assert re.match(r'^[a-zA-Z_-]+$', key)

-        if val:
-            assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)

         expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
         return node.find(expr)
 else:
     def find_xpath_attr(node, xpath, key, val=None):
         expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
         return node.find(expr)
 else:
     def find_xpath_attr(node, xpath, key, val=None):

-        # Here comes the crazy part: In 2.6, if the xpath is a unicode,
-        # .//node does not match if a node is a direct child of . !
-        if isinstance(xpath, compat_str):
-            xpath = xpath.encode('ascii')
-
-        for f in node.findall(xpath):
+        for f in node.findall(compat_xpath(xpath)):

             if key not in f.attrib:
                 continue
             if val is None or f.attrib.get(key) == val:
             if key not in f.attrib:
                 continue
             if val is None or f.attrib.get(key) == val:


@@ -196,9 +213,7 @@ def xpath_with_ns(path, ns_map):
 
 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     def _find_xpath(xpath):
 
 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     def _find_xpath(xpath):

-        if sys.version_info < (2, 7):  # Crazy 2.6
-            xpath = xpath.encode('ascii')
-        return node.find(xpath)
+        return node.find(compat_xpath(xpath))

 
     if isinstance(xpath, (str, compat_str)):
         n = _find_xpath(xpath)
 
     if isinstance(xpath, (str, compat_str)):
         n = _find_xpath(xpath)


@@ -257,9 +272,9 @@ def get_element_by_attribute(attribute, value, html):
 
     m = re.search(r'''(?xs)
         <([a-zA-Z0-9:._-]+)
 
     m = re.search(r'''(?xs)
         <([a-zA-Z0-9:._-]+)

-         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?

          \s+%s=['"]?%s['"]?
          \s+%s=['"]?%s['"]?

-         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?

         \s*>
         (?P<content>.*?)
         </\1>
         \s*>
         (?P<content>.*?)
         </\1>


@@ -275,6 +290,38 @@ def get_element_by_attribute(attribute, value, html):
     return unescapeHTML(res)
 
 
     return unescapeHTML(res)
 
 

+class HTMLAttributeParser(compat_HTMLParser):
+    """Trivial HTML parser to gather the attributes for a single element"""
+    def __init__(self):
+        self.attrs = {}
+        compat_HTMLParser.__init__(self)
+
+    def handle_starttag(self, tag, attrs):
+        self.attrs = dict(attrs)
+
+
+def extract_attributes(html_element):
+    """Given a string for an HTML element such as
+    <el
+         a="foo" B="bar" c="&98;az" d=boz
+         empty= noval entity="&amp;"
+         sq='"' dq="'"
+    >
+    Decode and return a dictionary of attributes.
+    {
+        'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
+        'empty': '', 'noval': None, 'entity': '&',
+        'sq': '"', 'dq': '\''
+    }.
+    NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
+    but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
+    """
+    parser = HTMLAttributeParser()
+    parser.feed(html_element)
+    parser.close()
+    return parser.attrs
+
+

 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
 
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
 


@@ -339,6 +386,8 @@ def sanitize_filename(s, restricted=False, is_id=False):
     Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
     """
     def replace_insane(char):
     Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
     """
     def replace_insane(char):

+        if restricted and char in ACCENT_CHARS:
+            return ACCENT_CHARS[char]

         if char == '?' or ord(char) < 32 or ord(char) == 127:
             return ''
         elif char == '"':
         if char == '?' or ord(char) < 32 or ord(char) == 127:
             return ''
         elif char == '"':


@@ -391,9 +440,12 @@ def sanitize_path(s):
 
 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
 # unwanted failures due to missing protocol
 
 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
 # unwanted failures due to missing protocol

+def sanitize_url(url):
+    return 'http:%s' % url if url.startswith('//') else url
+
+

 def sanitized_Request(url, *args, **kwargs):
 def sanitized_Request(url, *args, **kwargs):

-    return compat_urllib_request.Request(
-        'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
+    return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)

 
 
 def orderedSet(iterable):
 
 
 def orderedSet(iterable):


@@ -405,12 +457,19 @@ def orderedSet(iterable):
     return res
 
 
     return res
 
 

-def _htmlentity_transform(entity):
+def _htmlentity_transform(entity_with_semicolon):

     """Transforms an HTML entity to a character."""
     """Transforms an HTML entity to a character."""

+    entity = entity_with_semicolon[:-1]
+

     # Known non-numeric HTML entity
     if entity in compat_html_entities.name2codepoint:
         return compat_chr(compat_html_entities.name2codepoint[entity])
 
     # Known non-numeric HTML entity
     if entity in compat_html_entities.name2codepoint:
         return compat_chr(compat_html_entities.name2codepoint[entity])
 

+    # TODO: HTML5 allows entities without a semicolon. For example,
+    # '&Eacuteric' should be decoded as 'Éric'.
+    if entity_with_semicolon in compat_html_entities_html5:
+        return compat_html_entities_html5[entity_with_semicolon]
+

     mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
     if mobj is not None:
         numstr = mobj.group(1)
     mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
     if mobj is not None:
         numstr = mobj.group(1)


@@ -435,7 +494,7 @@ def unescapeHTML(s):
     assert type(s) == compat_str
 
     return re.sub(
     assert type(s) == compat_str
 
     return re.sub(

-        r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
+        r'&([^;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)

 
 
 def get_subprocess_encoding():
 
 
 def get_subprocess_encoding():


@@ -467,6 +526,10 @@ def encodeFilename(s, for_subprocess=False):
     if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
         return s
 
     if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
         return s
 

+    # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
+    if sys.platform.startswith('java'):
+        return s
+

     return s.encode(get_subprocess_encoding(), 'ignore')
 
 
     return s.encode(get_subprocess_encoding(), 'ignore')
 
 


@@ -712,8 +775,15 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
         self._params = params
 
     def http_open(self, req):
         self._params = params
 
     def http_open(self, req):

+        conn_class = compat_http_client.HTTPConnection
+
+        socks_proxy = req.headers.get('Ytdl-socks-proxy')
+        if socks_proxy:
+            conn_class = make_socks_conn_class(conn_class, socks_proxy)
+            del req.headers['Ytdl-socks-proxy']
+

         return self.do_open(functools.partial(
         return self.do_open(functools.partial(

-            _create_http_connection, self, compat_http_client.HTTPConnection, False),
+            _create_http_connection, self, conn_class, False),

             req)
 
     @staticmethod
             req)
 
     @staticmethod


@@ -745,12 +815,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
         # Substitute URL if any change after escaping
         if url != url_escaped:
 
         # Substitute URL if any change after escaping
         if url != url_escaped:

-            req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
-            new_req = req_type(
-                url_escaped, data=req.data, headers=req.headers,
-                origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
-            new_req.timeout = req.timeout
-            req = new_req
+            req = update_Request(req, url=url_escaped)

 
         for h, v in std_headers.items():
             # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
 
         for h, v in std_headers.items():
             # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275


@@ -804,9 +869,13 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
                 if sys.version_info >= (3, 0):
                     location = location.encode('iso-8859-1').decode('utf-8')
                 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
                 if sys.version_info >= (3, 0):
                     location = location.encode('iso-8859-1').decode('utf-8')

+                else:
+                    location = location.decode('utf-8')

                 location_escaped = escape_url(location)
                 if location != location_escaped:
                     del resp.headers['Location']
                 location_escaped = escape_url(location)
                 if location != location_escaped:
                     del resp.headers['Location']

+                    if sys.version_info < (3, 0):
+                        location_escaped = location_escaped.encode('utf-8')

                     resp.headers['Location'] = location_escaped
         return resp
 
                     resp.headers['Location'] = location_escaped
         return resp
 


@@ -814,6 +883,49 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     https_response = http_response
 
 
     https_response = http_response
 
 

+def make_socks_conn_class(base_class, socks_proxy):
+    assert issubclass(base_class, (
+        compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
+
+    url_components = compat_urlparse.urlparse(socks_proxy)
+    if url_components.scheme.lower() == 'socks5':
+        socks_type = ProxyType.SOCKS5
+    elif url_components.scheme.lower() in ('socks', 'socks4'):
+        socks_type = ProxyType.SOCKS4
+    elif url_components.scheme.lower() == 'socks4a':
+        socks_type = ProxyType.SOCKS4A
+
+    def unquote_if_non_empty(s):
+        if not s:
+            return s
+        return compat_urllib_parse_unquote_plus(s)
+
+    proxy_args = (
+        socks_type,
+        url_components.hostname, url_components.port or 1080,
+        True,  # Remote DNS
+        unquote_if_non_empty(url_components.username),
+        unquote_if_non_empty(url_components.password),
+    )
+
+    class SocksConnection(base_class):
+        def connect(self):
+            self.sock = sockssocket()
+            self.sock.setproxy(*proxy_args)
+            if type(self.timeout) in (int, float):
+                self.sock.settimeout(self.timeout)
+            self.sock.connect((self.host, self.port))
+
+            if isinstance(self, compat_http_client.HTTPSConnection):
+                if hasattr(self, '_context'):  # Python > 2.6
+                    self.sock = self._context.wrap_socket(
+                        self.sock, server_hostname=self.host)
+                else:
+                    self.sock = ssl.wrap_socket(self.sock)
+
+    return SocksConnection
+
+

 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
     def __init__(self, params, https_conn_class=None, *args, **kwargs):
         compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
     def __init__(self, params, https_conn_class=None, *args, **kwargs):
         compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)


@@ -822,12 +934,20 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
 
     def https_open(self, req):
         kwargs = {}
 
     def https_open(self, req):
         kwargs = {}

+        conn_class = self._https_conn_class
+

         if hasattr(self, '_context'):  # python > 2.6
             kwargs['context'] = self._context
         if hasattr(self, '_check_hostname'):  # python 3.x
             kwargs['check_hostname'] = self._check_hostname
         if hasattr(self, '_context'):  # python > 2.6
             kwargs['context'] = self._context
         if hasattr(self, '_check_hostname'):  # python 3.x
             kwargs['check_hostname'] = self._check_hostname

+
+        socks_proxy = req.headers.get('Ytdl-socks-proxy')
+        if socks_proxy:
+            conn_class = make_socks_conn_class(conn_class, socks_proxy)
+            del req.headers['Ytdl-socks-proxy']
+

         return self.do_open(functools.partial(
         return self.do_open(functools.partial(

-            _create_http_connection, self, self._https_conn_class, True),
+            _create_http_connection, self, conn_class, True),

             req, **kwargs)
 
 
             req, **kwargs)
 
 


@@ -905,9 +1025,9 @@ def unified_strdate(date_str, day_first=True):
         '%d %b %Y',
         '%B %d %Y',
         '%b %d %Y',
         '%d %b %Y',
         '%B %d %Y',
         '%b %d %Y',

-        '%b %dst %Y %I:%M%p',
-        '%b %dnd %Y %I:%M%p',
-        '%b %dth %Y %I:%M%p',
+        '%b %dst %Y %I:%M',
+        '%b %dnd %Y %I:%M',
+        '%b %dth %Y %I:%M',

         '%Y %m %d',
         '%Y-%m-%d',
         '%Y/%m/%d',
         '%Y %m %d',
         '%Y-%m-%d',
         '%Y/%m/%d',


@@ -927,6 +1047,7 @@ def unified_strdate(date_str, day_first=True):
         format_expressions.extend([
             '%d-%m-%Y',
             '%d.%m.%Y',
         format_expressions.extend([
             '%d-%m-%Y',
             '%d.%m.%Y',

+            '%d.%m.%y',

             '%d/%m/%Y',
             '%d/%m/%y',
             '%d/%m/%Y %H:%M:%S',
             '%d/%m/%Y',
             '%d/%m/%y',
             '%d/%m/%Y %H:%M:%S',


@@ -947,7 +1068,10 @@ def unified_strdate(date_str, day_first=True):
     if upload_date is None:
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:
     if upload_date is None:
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:

-            upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
+            try:
+                upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
+            except ValueError:
+                pass

     if upload_date is not None:
         return compat_str(upload_date)
 
     if upload_date is not None:
         return compat_str(upload_date)
 


@@ -1158,7 +1282,7 @@ def bytes_to_intlist(bs):
 def intlist_to_bytes(xs):
     if not xs:
         return b''
 def intlist_to_bytes(xs):
     if not xs:
         return b''

-    return struct_pack('%dB' % len(xs), *xs)
+    return compat_struct_pack('%dB' % len(xs), *xs)

 
 
 # Cross-platform file locking
 
 
 # Cross-platform file locking


@@ -1217,13 +1341,23 @@ if sys.platform == 'win32':
             raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
 
 else:
             raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
 
 else:

-    import fcntl
+    # Some platforms, such as Jython, is missing fcntl
+    try:
+        import fcntl

 
 

-    def _lock_file(f, exclusive):
-        fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
+        def _lock_file(f, exclusive):
+            fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)

 
 

-    def _unlock_file(f):
-        fcntl.flock(f, fcntl.LOCK_UN)
+        def _unlock_file(f):
+            fcntl.flock(f, fcntl.LOCK_UN)
+    except ImportError:
+        UNSUPPORTED_MSG = 'file locking is not supported on this platform'
+
+        def _lock_file(f, exclusive):
+            raise IOError(UNSUPPORTED_MSG)
+
+        def _unlock_file(f):
+            raise IOError(UNSUPPORTED_MSG)

 
 
 class locked_file(object):
 
 
 class locked_file(object):


@@ -1276,7 +1410,7 @@ def shell_quote(args):
 def smuggle_url(url, data):
     """ Pass additional data in a URL for internal use. """
 
 def smuggle_url(url, data):
     """ Pass additional data in a URL for internal use. """
 

-    sdata = compat_urllib_parse.urlencode(
+    sdata = compat_urllib_parse_urlencode(

         {'__youtubedl_smuggle': json.dumps(data)})
     return url + '#' + sdata
 
         {'__youtubedl_smuggle': json.dumps(data)})
     return url + '#' + sdata
 


@@ -1304,6 +1438,17 @@ def format_bytes(bytes):
     return '%.2f%s' % (converted, suffix)
 
 
     return '%.2f%s' % (converted, suffix)
 
 

+def lookup_unit_table(unit_table, s):
+    units_re = '|'.join(re.escape(u) for u in unit_table)
+    m = re.match(
+        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
+    if not m:
+        return None
+    num_str = m.group('num').replace(',', '.')
+    mult = unit_table[m.group('unit')]
+    return int(float(num_str) * mult)
+
+

 def parse_filesize(s):
     if s is None:
         return None
 def parse_filesize(s):
     if s is None:
         return None


@@ -1347,15 +1492,28 @@ def parse_filesize(s):
         'Yb': 1000 ** 8,
     }
 
         'Yb': 1000 ** 8,
     }
 

-    units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
-    m = re.match(
-        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
-    if not m:
+    return lookup_unit_table(_UNIT_TABLE, s)
+
+
+def parse_count(s):
+    if s is None:

         return None
 
         return None
 

-    num_str = m.group('num').replace(',', '.')
-    mult = _UNIT_TABLE[m.group('unit')]
-    return int(float(num_str) * mult)
+    s = s.strip()
+
+    if re.match(r'^[\d,.]+$', s):
+        return str_to_int(s)
+
+    _UNIT_TABLE = {
+        'k': 1000,
+        'K': 1000,
+        'm': 1000 ** 2,
+        'M': 1000 ** 2,
+        'kk': 1000 ** 2,
+        'KK': 1000 ** 2,
+    }
+
+    return lookup_unit_table(_UNIT_TABLE, s)

 
 
 def month_by_name(name):
 
 
 def month_by_name(name):


@@ -1387,6 +1545,12 @@ def fix_xml_ampersands(xml_str):
 
 def setproctitle(title):
     assert isinstance(title, compat_str)
 
 def setproctitle(title):
     assert isinstance(title, compat_str)

+
+    # ctypes in Jython is not complete
+    # http://bugs.jython.org/issue2148
+    if sys.platform.startswith('java'):
+        return
+

     try:
         libc = ctypes.cdll.LoadLibrary('libc.so.6')
     except OSError:
     try:
         libc = ctypes.cdll.LoadLibrary('libc.so.6')
     except OSError:


@@ -1401,15 +1565,11 @@ def setproctitle(title):
 
 
 def remove_start(s, start):
 
 
 def remove_start(s, start):

-    if s.startswith(start):
-        return s[len(start):]
-    return s
+    return s[len(start):] if s is not None and s.startswith(start) else s

 
 
 def remove_end(s, end):
 
 
 def remove_end(s, end):

-    if s.endswith(end):
-        return s[:-len(end)]
-    return s
+    return s[:-len(end)] if s is not None and s.endswith(end) else s

 
 
 def remove_quotes(s):
 
 
 def remove_quotes(s):


@@ -1472,44 +1632,46 @@ def parse_duration(s):
 
     s = s.strip()
 
 
     s = s.strip()
 

-    m = re.match(
-        r'''(?ix)(?:P?T)?
-        (?:
-            (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
-            (?P<only_hours>[0-9.]+)\s*(?:hours?)|
-
-            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
-            (?:
+    days, hours, mins, secs, ms = [None] * 5
+    m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?$', s)
+    if m:
+        days, hours, mins, secs, ms = m.groups()
+    else:
+        m = re.match(
+            r'''(?ix)(?:P?T)?

                 (?:
                 (?:

-                    (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
-                    (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
+                    (?P<days>[0-9]+)\s*d(?:ays?)?\s*

                 )?
                 )?

-                (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
-            )?
-            (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
-        )$''', s)
-    if not m:
-        return None
-    res = 0
-    if m.group('only_mins'):
-        return float_or_none(m.group('only_mins'), invscale=60)
-    if m.group('only_hours'):
-        return float_or_none(m.group('only_hours'), invscale=60 * 60)
-    if m.group('secs'):
-        res += int(m.group('secs'))
-    if m.group('mins_reversed'):
-        res += int(m.group('mins_reversed')) * 60
-    if m.group('mins'):
-        res += int(m.group('mins')) * 60
-    if m.group('hours'):
-        res += int(m.group('hours')) * 60 * 60
-    if m.group('hours_reversed'):
-        res += int(m.group('hours_reversed')) * 60 * 60
-    if m.group('days'):
-        res += int(m.group('days')) * 24 * 60 * 60
-    if m.group('ms'):
-        res += float(m.group('ms'))
-    return res
+                (?:
+                    (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
+                )?
+                (?:
+                    (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
+                )?
+                (?:
+                    (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
+                )?$''', s)
+        if m:
+            days, hours, mins, secs, ms = m.groups()
+        else:
+            m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)$', s)
+            if m:
+                hours, mins = m.groups()
+            else:
+                return None
+
+    duration = 0
+    if secs:
+        duration += float(secs)
+    if mins:
+        duration += float(mins) * 60
+    if hours:
+        duration += float(hours) * 60 * 60
+    if days:
+        duration += float(days) * 24 * 60 * 60
+    if ms:
+        duration += float(ms)
+    return duration

 
 
 def prepend_extension(filename, ext, expected_real_ext=None):
 
 
 def prepend_extension(filename, ext, expected_real_ext=None):


@@ -1570,9 +1732,12 @@ class PagedList(object):
 
 
 class OnDemandPagedList(PagedList):
 
 
 class OnDemandPagedList(PagedList):

-    def __init__(self, pagefunc, pagesize):
+    def __init__(self, pagefunc, pagesize, use_cache=False):

         self._pagefunc = pagefunc
         self._pagesize = pagesize
         self._pagefunc = pagefunc
         self._pagesize = pagesize

+        self._use_cache = use_cache
+        if use_cache:
+            self._cache = {}

 
     def getslice(self, start=0, end=None):
         res = []
 
     def getslice(self, start=0, end=None):
         res = []


@@ -1582,7 +1747,13 @@ class OnDemandPagedList(PagedList):
             if start >= nextfirstid:
                 continue
 
             if start >= nextfirstid:
                 continue
 

-            page_results = list(self._pagefunc(pagenum))
+            page_results = None
+            if self._use_cache:
+                page_results = self._cache.get(pagenum)
+            if page_results is None:
+                page_results = list(self._pagefunc(pagenum))
+            if self._use_cache:
+                self._cache[pagenum] = page_results

 
             startv = (
                 start % self._pagesize
 
             startv = (
                 start % self._pagesize


@@ -1668,29 +1839,13 @@ def escape_url(url):
     """Escape URL as suggested by RFC 3986"""
     url_parsed = compat_urllib_parse_urlparse(url)
     return url_parsed._replace(
     """Escape URL as suggested by RFC 3986"""
     url_parsed = compat_urllib_parse_urlparse(url)
     return url_parsed._replace(

+        netloc=url_parsed.netloc.encode('idna').decode('ascii'),

         path=escape_rfc3986(url_parsed.path),
         params=escape_rfc3986(url_parsed.params),
         query=escape_rfc3986(url_parsed.query),
         fragment=escape_rfc3986(url_parsed.fragment)
     ).geturl()
 
         path=escape_rfc3986(url_parsed.path),
         params=escape_rfc3986(url_parsed.params),
         query=escape_rfc3986(url_parsed.query),
         fragment=escape_rfc3986(url_parsed.fragment)
     ).geturl()
 

-try:
-    struct.pack('!I', 0)
-except TypeError:
-    # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
-    def struct_pack(spec, *args):
-        if isinstance(spec, compat_str):
-            spec = spec.encode('ascii')
-        return struct.pack(spec, *args)
-
-    def struct_unpack(spec, *args):
-        if isinstance(spec, compat_str):
-            spec = spec.encode('ascii')
-        return struct.unpack(spec, *args)
-else:
-    struct_pack = struct.pack
-    struct_unpack = struct.unpack
-

 
 def read_batch_urls(batch_fd):
     def fixup(url):
 
 def read_batch_urls(batch_fd):
     def fixup(url):


@@ -1709,13 +1864,31 @@ def read_batch_urls(batch_fd):
 
 
 def urlencode_postdata(*args, **kargs):
 
 
 def urlencode_postdata(*args, **kargs):

-    return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
+    return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')

 
 
 
 

-def encode_dict(d, encoding='utf-8'):
-    def encode(v):
-        return v.encode(encoding) if isinstance(v, compat_basestring) else v
-    return dict((encode(k), encode(v)) for k, v in d.items())
+def update_url_query(url, query):
+    if not query:
+        return url
+    parsed_url = compat_urlparse.urlparse(url)
+    qs = compat_parse_qs(parsed_url.query)
+    qs.update(query)
+    return compat_urlparse.urlunparse(parsed_url._replace(
+        query=compat_urllib_parse_urlencode(qs, True)))
+
+
+def update_Request(req, url=None, data=None, headers={}, query={}):
+    req_headers = req.headers.copy()
+    req_headers.update(headers)
+    req_data = data or req.data
+    req_url = update_url_query(url or req.get_full_url(), query)
+    req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
+    new_req = req_type(
+        req_url, data=req_data, headers=req_headers,
+        origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
+    if hasattr(req, 'timeout'):
+        new_req.timeout = req.timeout
+    return new_req

 
 
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
 
 
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):


@@ -1728,6 +1901,16 @@ def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     return d.get(key_or_keys, default)
 
 
     return d.get(key_or_keys, default)
 
 

+def try_get(src, getter, expected_type=None):
+    try:
+        v = getter(src)
+    except (AttributeError, KeyError, TypeError, IndexError):
+        pass
+    else:
+        if expected_type is None or isinstance(v, expected_type):
+            return v
+
+

 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
     return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
 
 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
     return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
 


@@ -1750,7 +1933,7 @@ def parse_age_limit(s):
 
 def strip_jsonp(code):
     return re.sub(
 
 def strip_jsonp(code):
     return re.sub(

-        r'(?s)^[a-zA-Z0-9_.]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
+        r'(?s)^[a-zA-Z0-9_.$]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)

 
 
 def js_to_json(code):
 
 
 def js_to_json(code):


@@ -1758,24 +1941,38 @@ def js_to_json(code):
         v = m.group(0)
         if v in ('true', 'false', 'null'):
             return v
         v = m.group(0)
         if v in ('true', 'false', 'null'):
             return v

-        if v.startswith('"'):
-            v = re.sub(r"\\'", "'", v[1:-1])
-        elif v.startswith("'"):
-            v = v[1:-1]
-            v = re.sub(r"\\\\|\\'|\"", lambda m: {
-                '\\\\': '\\\\',
-                "\\'": "'",
+        elif v.startswith('/*') or v == ',':
+            return ""
+
+        if v[0] in ("'", '"'):
+            v = re.sub(r'(?s)\\.|"', lambda m: {

                 '"': '\\"',
                 '"': '\\"',

-            }[m.group(0)], v)
+                "\\'": "'",
+                '\\\n': '',
+                '\\x': '\\u00',
+            }.get(m.group(0), m.group(0)), v[1:-1])
+
+        INTEGER_TABLE = (
+            (r'^0[xX][0-9a-fA-F]+', 16),
+            (r'^0+[0-7]+', 8),
+        )
+
+        for regex, base in INTEGER_TABLE:
+            im = re.match(regex, v)
+            if im:
+                i = int(im.group(0), base)
+                return '"%d":' % i if v.endswith(':') else '%d' % i
+

         return '"%s"' % v
 
         return '"%s"' % v
 

-    res = re.sub(r'''(?x)
-        "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
-        '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
-        [a-zA-Z_][.a-zA-Z_0-9]*
+    return re.sub(r'''(?sx)
+        "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
+        '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
+        /\*.*?\*/|,(?=\s*[\]}])|
+        [a-zA-Z_][.a-zA-Z_0-9]*|
+        \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
+        [0-9]+(?=\s*:)

         ''', fix_kv, code)
         ''', fix_kv, code)

-    res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
-    return res

 
 
 def qualities(quality_ids):
 
 
 def qualities(quality_ids):


@@ -1823,7 +2020,7 @@ def ytdl_is_updateable():
 
 def args_to_str(args):
     # Get a short string representation for a subprocess command
 
 def args_to_str(args):
     # Get a short string representation for a subprocess command

-    return ' '.join(shlex_quote(a) for a in args)
+    return ' '.join(compat_shlex_quote(a) for a in args)

 
 
 def error_to_compat_str(err):
 
 
 def error_to_compat_str(err):


@@ -1836,8 +2033,14 @@ def error_to_compat_str(err):
 
 
 def mimetype2ext(mt):
 
 
 def mimetype2ext(mt):

+    if mt is None:
+        return None
+

     ext = {
         'audio/mp4': 'm4a',
     ext = {
         'audio/mp4': 'm4a',

+        # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
+        # it's the most popular one
+        'audio/mpeg': 'mp3',

     }.get(mt)
     if ext is not None:
         return ext
     }.get(mt)
     if ext is not None:
         return ext


@@ -1858,11 +2061,7 @@ def mimetype2ext(mt):
 
 
 def urlhandle_detect_ext(url_handle):
 
 
 def urlhandle_detect_ext(url_handle):

-    try:
-        url_handle.headers
-        getheader = lambda h: url_handle.headers[h]
-    except AttributeError:  # Python < 3
-        getheader = url_handle.info().getheader
+    getheader = url_handle.headers.get

 
     cd = getheader('Content-Disposition')
     if cd:
 
     cd = getheader('Content-Disposition')
     if cd:


@@ -2036,6 +2235,7 @@ def dfxp2srt(dfxp_data):
     _x = functools.partial(xpath_with_ns, ns_map={
         'ttml': 'http://www.w3.org/ns/ttml',
         'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
     _x = functools.partial(xpath_with_ns, ns_map={
         'ttml': 'http://www.w3.org/ns/ttml',
         'ttaf1': 'http://www.w3.org/2006/10/ttaf1',

+        'ttaf1_0604': 'http://www.w3.org/2006/04/ttaf1',

     })
 
     class TTMLPElementParser(object):
     })
 
     class TTMLPElementParser(object):


@@ -2062,7 +2262,7 @@ def dfxp2srt(dfxp_data):
 
     dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
     out = []
 
     dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
     out = []

-    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
+    paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall(_x('.//ttaf1_0604:p')) or dfxp.findall('.//p')

 
     if not paras:
         raise ValueError('Invalid dfxp/TTML subtitle')
 
     if not paras:
         raise ValueError('Invalid dfxp/TTML subtitle')


@@ -2591,6 +2791,10 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
 
         if proxy == '__noproxy__':
             return None  # No Proxy
 
         if proxy == '__noproxy__':
             return None  # No Proxy

+        if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
+            req.add_header('Ytdl-socks-proxy', proxy)
+            # youtube-dl's http/https handlers do wrapping the socket with socks
+            return None

         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
 
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
 


@@ -2610,3 +2814,50 @@ def ohdave_rsa_encrypt(data, exponent, modulus):
     payload = int(binascii.hexlify(data[::-1]), 16)
     encrypted = pow(payload, exponent, modulus)
     return '%x' % encrypted
     payload = int(binascii.hexlify(data[::-1]), 16)
     encrypted = pow(payload, exponent, modulus)
     return '%x' % encrypted

+
+
+def encode_base_n(num, n, table=None):
+    FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+    if not table:
+        table = FULL_TABLE[:n]
+
+    if n > len(table):
+        raise ValueError('base %d exceeds table length %d' % (n, len(table)))
+
+    if num == 0:
+        return table[0]
+
+    ret = ''
+    while num:
+        ret = table[num % n] + ret
+        num = num // n
+    return ret
+
+
+def decode_packed_codes(code):
+    mobj = re.search(
+        r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)",
+        code)
+    obfucasted_code, base, count, symbols = mobj.groups()
+    base = int(base)
+    count = int(count)
+    symbols = symbols.split('|')
+    symbol_table = {}
+
+    while count:
+        count -= 1
+        base_n_count = encode_base_n(count, base)
+        symbol_table[base_n_count] = symbols[count] or base_n_count
+
+    return re.sub(
+        r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
+        obfucasted_code)
+
+
+def parse_m3u8_attributes(attrib):
+    info = {}
+    for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
+        if val.startswith('"'):
+            val = val[1:-1]
+        info[key] = val
+    return info