Initiate release.

[youtubedl] / youtube_dl / utils.py

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b97e62ae9307f7e2380db7ec9c723e8ae8517708..e40b367c255719046bf2d5dd2fd63bc6bb2e4d8a 100644 (file)
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -91,11 +91,9 @@ except ImportError:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
 try:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
 try:

-    from urllib.parse import parse_qs as compat_parse_qs
-except ImportError: # Python 2
-    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
-    # Python 2's version is apparently totally broken
-    def _unquote(string, encoding='utf-8', errors='replace'):
+    from urllib.parse import unquote as compat_urllib_parse_unquote
+except ImportError:
+    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):

         if string == '':
             return string
         res = string.split('%')
         if string == '':
             return string
         res = string.split('%')


@@ -130,6 +128,13 @@ except ImportError: # Python 2
             string += pct_sequence.decode(encoding, errors)
         return string
 
             string += pct_sequence.decode(encoding, errors)
         return string
 

+
+try:
+    from urllib.parse import parse_qs as compat_parse_qs
+except ImportError: # Python 2
+    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+    # Python 2's version is apparently totally broken
+

     def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
                 encoding='utf-8', errors='replace'):
         qs, _coerce_result = qs, unicode
     def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
                 encoding='utf-8', errors='replace'):
         qs, _coerce_result = qs, unicode


@@ -149,10 +154,12 @@ except ImportError: # Python 2
                     continue
             if len(nv[1]) or keep_blank_values:
                 name = nv[0].replace('+', ' ')
                     continue
             if len(nv[1]) or keep_blank_values:
                 name = nv[0].replace('+', ' ')

-                name = _unquote(name, encoding=encoding, errors=errors)
+                name = compat_urllib_parse_unquote(
+                    name, encoding=encoding, errors=errors)

                 name = _coerce_result(name)
                 value = nv[1].replace('+', ' ')
                 name = _coerce_result(name)
                 value = nv[1].replace('+', ' ')

-                value = _unquote(value, encoding=encoding, errors=errors)
+                value = compat_urllib_parse_unquote(
+                    value, encoding=encoding, errors=errors)

                 value = _coerce_result(value)
                 r.append((name, value))
         return r
                 value = _coerce_result(value)
                 r.append((name, value))
         return r


@@ -235,8 +242,8 @@ else:
 if sys.version_info >= (2,7):
     def find_xpath_attr(node, xpath, key, val):
         """ Find the xpath xpath[@key=val] """
 if sys.version_info >= (2,7):
     def find_xpath_attr(node, xpath, key, val):
         """ Find the xpath xpath[@key=val] """

-        assert re.match(r'^[a-zA-Z]+$', key)
-        assert re.match(r'^[a-zA-Z0-9@\s:._]*$', val)
+        assert re.match(r'^[a-zA-Z-]+$', key)
+        assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)

         expr = xpath + u"[@%s='%s']" % (key, val)
         return node.find(expr)
 else:
         expr = xpath + u"[@%s='%s']" % (key, val)
         return node.find(expr)
 else:


@@ -775,7 +782,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     https_response = http_response
 
 
     https_response = http_response
 
 

-def parse_iso8601(date_str):
+def parse_iso8601(date_str, delimiter='T'):

     """ Return a UNIX timestamp from the given date """
 
     if date_str is None:
     """ Return a UNIX timestamp from the given date """
 
     if date_str is None:


@@ -795,8 +802,8 @@ def parse_iso8601(date_str):
             timezone = datetime.timedelta(
                 hours=sign * int(m.group('hours')),
                 minutes=sign * int(m.group('minutes')))
             timezone = datetime.timedelta(
                 hours=sign * int(m.group('hours')),
                 minutes=sign * int(m.group('minutes')))

-
-    dt = datetime.datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S') - timezone
+    date_format =  '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
+    dt = datetime.datetime.strptime(date_str, date_format) - timezone

     return calendar.timegm(dt.timetuple())
 
 
     return calendar.timegm(dt.timetuple())
 
 


@@ -816,6 +823,9 @@ def unified_strdate(date_str):
         '%d %b %Y',
         '%B %d %Y',
         '%b %d %Y',
         '%d %b %Y',
         '%B %d %Y',
         '%b %d %Y',

+        '%b %dst %Y %I:%M%p',
+        '%b %dnd %Y %I:%M%p',
+        '%b %dth %Y %I:%M%p',

         '%Y-%m-%d',
         '%d.%m.%Y',
         '%d/%m/%Y',
         '%Y-%m-%d',
         '%d.%m.%Y',
         '%d/%m/%Y',


@@ -842,6 +852,8 @@ def unified_strdate(date_str):
     return upload_date
 
 def determine_ext(url, default_ext=u'unknown_video'):
     return upload_date
 
 def determine_ext(url, default_ext=u'unknown_video'):

+    if url is None:
+        return default_ext

     guess = url.partition(u'?')[0].rpartition(u'.')[2]
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
     guess = url.partition(u'?')[0].rpartition(u'.')[2]
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess


@@ -1190,11 +1202,6 @@ def format_bytes(bytes):
     return u'%.2f%s' % (converted, suffix)
 
 
     return u'%.2f%s' % (converted, suffix)
 
 

-def str_to_int(int_str):
-    int_str = re.sub(r'[,\.]', u'', int_str)
-    return int(int_str)
-
-

 def get_term_width():
     columns = os.environ.get('COLUMNS', None)
     if columns:
 def get_term_width():
     columns = os.environ.get('COLUMNS', None)
     if columns:


@@ -1262,15 +1269,22 @@ class HEADRequest(compat_urllib_request.Request):
         return "HEAD"
 
 
         return "HEAD"
 
 

-def int_or_none(v, scale=1, default=None, get_attr=None):
+def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):

     if get_attr:
         if v is not None:
             v = getattr(v, get_attr, None)
     if get_attr:
         if v is not None:
             v = getattr(v, get_attr, None)

-    return default if v is None else (int(v) // scale)
+    return default if v is None else (int(v) * invscale // scale)
+
+
+def str_to_int(int_str):
+    if int_str is None:
+        return None
+    int_str = re.sub(r'[,\.]', u'', int_str)
+    return int(int_str)

 
 
 
 

-def float_or_none(v, scale=1, default=None):
-    return default if v is None else (float(v) / scale)
+def float_or_none(v, scale=1, invscale=1, default=None):
+    return default if v is None else (float(v) * invscale / scale)

 
 
 def parse_duration(s):
 
 
 def parse_duration(s):


@@ -1425,7 +1439,7 @@ US_RATINGS = {
 
 
 def strip_jsonp(code):
 
 
 def strip_jsonp(code):

-    return re.sub(r'(?s)^[a-zA-Z_]+\s*\(\s*(.*)\);\s*?\s*$', r'\1', code)
+    return re.sub(r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?\s*$', r'\1', code)

 
 
 def qualities(quality_ids):
 
 
 def qualities(quality_ids):