Prepare to upload.

[youtubedl] / youtube_dl / utils.py

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7987572419a398349db4f73d2901fe324f3a401a..38262bee4cd08f97e4f5009d7066d1466bde25a4 100644 (file)
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -46,6 +46,7 @@ from .compat import (
     compat_html_entities,
     compat_html_entities_html5,
     compat_http_client,
     compat_html_entities,
     compat_html_entities_html5,
     compat_http_client,

+    compat_integer_types,

     compat_kwargs,
     compat_os_name,
     compat_parse_qs,
     compat_kwargs,
     compat_os_name,
     compat_parse_qs,


@@ -1718,13 +1719,16 @@ DATE_FORMATS = (
     '%B %d %Y',
     '%B %dst %Y',
     '%B %dnd %Y',
     '%B %d %Y',
     '%B %dst %Y',
     '%B %dnd %Y',

+    '%B %drd %Y',

     '%B %dth %Y',
     '%b %d %Y',
     '%b %dst %Y',
     '%b %dnd %Y',
     '%B %dth %Y',
     '%b %d %Y',
     '%b %dst %Y',
     '%b %dnd %Y',

+    '%b %drd %Y',

     '%b %dth %Y',
     '%b %dst %Y %I:%M',
     '%b %dnd %Y %I:%M',
     '%b %dth %Y',
     '%b %dst %Y %I:%M',
     '%b %dnd %Y %I:%M',

+    '%b %drd %Y %I:%M',

     '%b %dth %Y %I:%M',
     '%Y %m %d',
     '%Y-%m-%d',
     '%b %dth %Y %I:%M',
     '%Y %m %d',
     '%Y-%m-%d',


@@ -2725,6 +2729,11 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
 
 
 class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
 
 
 class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):

+    """
+    See [1] for cookie file format.
+
+    1. https://curl.haxx.se/docs/http-cookies.html
+    """

     _HTTPONLY_PREFIX = '#HttpOnly_'
 
     def save(self, filename=None, ignore_discard=False, ignore_expires=False):
     _HTTPONLY_PREFIX = '#HttpOnly_'
 
     def save(self, filename=None, ignore_discard=False, ignore_expires=False):


@@ -2791,6 +2800,15 @@ class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
     https_response = http_response
 
 
     https_response = http_response
 
 

+class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
+    if sys.version_info[0] < 3:
+        def redirect_request(self, req, fp, code, msg, headers, newurl):
+            # On python 2 urlh.geturl() may sometimes return redirect URL
+            # as byte string instead of unicode. This workaround allows
+            # to force it always return unicode.
+            return compat_urllib_request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, compat_str(newurl))
+
+

 def extract_timezone(date_str):
     m = re.search(
         r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
 def extract_timezone(date_str):
     m = re.search(
         r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',


@@ -2906,8 +2924,8 @@ def determine_ext(url, default_ext='unknown_video'):
         return default_ext
 
 
         return default_ext
 
 

-def subtitles_filename(filename, sub_lang, sub_format):
-    return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
+def subtitles_filename(filename, sub_lang, sub_format, expected_real_ext=None):
+    return replace_extension(filename, sub_lang + '.' + sub_format, expected_real_ext)

 
 
 def date_from_str(date_str):
 
 
 def date_from_str(date_str):


@@ -3516,10 +3534,11 @@ def str_or_none(v, default=None):
 
 def str_to_int(int_str):
     """ A more relaxed version of int_or_none """
 
 def str_to_int(int_str):
     """ A more relaxed version of int_or_none """

-    if int_str is None:
-        return None
-    int_str = re.sub(r'[,\.\+]', '', int_str)
-    return int(int_str)
+    if isinstance(int_str, compat_integer_types):
+        return int_str
+    elif isinstance(int_str, compat_str):
+        int_str = re.sub(r'[,\.\+]', '', int_str)
+        return int_or_none(int_str)

 
 
 def float_or_none(v, scale=1, invscale=1, default=None):
 
 
 def float_or_none(v, scale=1, invscale=1, default=None):


@@ -4979,7 +4998,7 @@ class ISO3166Utils(object):
 class GeoUtils(object):
     # Major IPv4 address blocks per country
     _country_ip_map = {
 class GeoUtils(object):
     # Major IPv4 address blocks per country
     _country_ip_map = {

-        'AD': '85.94.160.0/19',
+        'AD': '46.172.224.0/19',

         'AE': '94.200.0.0/13',
         'AF': '149.54.0.0/17',
         'AG': '209.59.64.0/18',
         'AE': '94.200.0.0/13',
         'AF': '149.54.0.0/17',
         'AG': '209.59.64.0/18',


@@ -4987,28 +5006,30 @@ class GeoUtils(object):
         'AL': '46.99.0.0/16',
         'AM': '46.70.0.0/15',
         'AO': '105.168.0.0/13',
         'AL': '46.99.0.0/16',
         'AM': '46.70.0.0/15',
         'AO': '105.168.0.0/13',

-        'AP': '159.117.192.0/21',
+        'AP': '182.50.184.0/21',
+        'AQ': '23.154.160.0/24',

         'AR': '181.0.0.0/12',
         'AS': '202.70.112.0/20',
         'AR': '181.0.0.0/12',
         'AS': '202.70.112.0/20',

-        'AT': '84.112.0.0/13',
+        'AT': '77.116.0.0/14',

         'AU': '1.128.0.0/11',
         'AW': '181.41.0.0/18',
         'AU': '1.128.0.0/11',
         'AW': '181.41.0.0/18',

-        'AZ': '5.191.0.0/16',
+        'AX': '185.217.4.0/22',
+        'AZ': '5.197.0.0/16',

         'BA': '31.176.128.0/17',
         'BB': '65.48.128.0/17',
         'BD': '114.130.0.0/16',
         'BE': '57.0.0.0/8',
         'BA': '31.176.128.0/17',
         'BB': '65.48.128.0/17',
         'BD': '114.130.0.0/16',
         'BE': '57.0.0.0/8',

-        'BF': '129.45.128.0/17',
+        'BF': '102.178.0.0/15',

         'BG': '95.42.0.0/15',
         'BH': '37.131.0.0/17',
         'BI': '154.117.192.0/18',
         'BJ': '137.255.0.0/16',
         'BG': '95.42.0.0/15',
         'BH': '37.131.0.0/17',
         'BI': '154.117.192.0/18',
         'BJ': '137.255.0.0/16',

-        'BL': '192.131.134.0/24',
+        'BL': '185.212.72.0/23',

         'BM': '196.12.64.0/18',
         'BN': '156.31.0.0/16',
         'BO': '161.56.0.0/16',
         'BQ': '161.0.80.0/20',
         'BM': '196.12.64.0/18',
         'BN': '156.31.0.0/16',
         'BO': '161.56.0.0/16',
         'BQ': '161.0.80.0/20',

-        'BR': '152.240.0.0/12',
+        'BR': '191.128.0.0/12',

         'BS': '24.51.64.0/18',
         'BT': '119.2.96.0/19',
         'BW': '168.167.0.0/16',
         'BS': '24.51.64.0/18',
         'BT': '119.2.96.0/19',
         'BW': '168.167.0.0/16',


@@ -5016,20 +5037,20 @@ class GeoUtils(object):
         'BZ': '179.42.192.0/18',
         'CA': '99.224.0.0/11',
         'CD': '41.243.0.0/16',
         'BZ': '179.42.192.0/18',
         'CA': '99.224.0.0/11',
         'CD': '41.243.0.0/16',

-        'CF': '196.32.200.0/21',
-        'CG': '197.214.128.0/17',
+        'CF': '197.242.176.0/21',
+        'CG': '160.113.0.0/16',

         'CH': '85.0.0.0/13',
         'CH': '85.0.0.0/13',

-        'CI': '154.232.0.0/14',
+        'CI': '102.136.0.0/14',

         'CK': '202.65.32.0/19',
         'CL': '152.172.0.0/14',
         'CK': '202.65.32.0/19',
         'CL': '152.172.0.0/14',

-        'CM': '165.210.0.0/15',
+        'CM': '102.244.0.0/14',

         'CN': '36.128.0.0/10',
         'CO': '181.240.0.0/12',
         'CR': '201.192.0.0/12',
         'CU': '152.206.0.0/15',
         'CV': '165.90.96.0/19',
         'CW': '190.88.128.0/17',
         'CN': '36.128.0.0/10',
         'CO': '181.240.0.0/12',
         'CR': '201.192.0.0/12',
         'CU': '152.206.0.0/15',
         'CV': '165.90.96.0/19',
         'CW': '190.88.128.0/17',

-        'CY': '46.198.0.0/15',
+        'CY': '31.153.0.0/16',

         'CZ': '88.100.0.0/14',
         'DE': '53.0.0.0/8',
         'DJ': '197.241.0.0/17',
         'CZ': '88.100.0.0/14',
         'DE': '53.0.0.0/8',
         'DJ': '197.241.0.0/17',


@@ -5046,6 +5067,7 @@ class GeoUtils(object):
         'EU': '2.16.0.0/13',
         'FI': '91.152.0.0/13',
         'FJ': '144.120.0.0/16',
         'EU': '2.16.0.0/13',
         'FI': '91.152.0.0/13',
         'FJ': '144.120.0.0/16',

+        'FK': '80.73.208.0/21',

         'FM': '119.252.112.0/20',
         'FO': '88.85.32.0/19',
         'FR': '90.0.0.0/9',
         'FM': '119.252.112.0/20',
         'FO': '88.85.32.0/19',
         'FR': '90.0.0.0/9',


@@ -5055,8 +5077,8 @@ class GeoUtils(object):
         'GE': '31.146.0.0/16',
         'GF': '161.22.64.0/18',
         'GG': '62.68.160.0/19',
         'GE': '31.146.0.0/16',
         'GF': '161.22.64.0/18',
         'GG': '62.68.160.0/19',

-        'GH': '45.208.0.0/14',
-        'GI': '85.115.128.0/19',
+        'GH': '154.160.0.0/12',
+        'GI': '95.164.0.0/16',

         'GL': '88.83.0.0/19',
         'GM': '160.182.0.0/15',
         'GN': '197.149.192.0/18',
         'GL': '88.83.0.0/19',
         'GM': '160.182.0.0/15',
         'GN': '197.149.192.0/18',


@@ -5085,13 +5107,13 @@ class GeoUtils(object):
         'JE': '87.244.64.0/18',
         'JM': '72.27.0.0/17',
         'JO': '176.29.0.0/16',
         'JE': '87.244.64.0/18',
         'JM': '72.27.0.0/17',
         'JO': '176.29.0.0/16',

-        'JP': '126.0.0.0/8',
+        'JP': '133.0.0.0/8',

         'KE': '105.48.0.0/12',
         'KG': '158.181.128.0/17',
         'KH': '36.37.128.0/17',
         'KI': '103.25.140.0/22',
         'KM': '197.255.224.0/20',
         'KE': '105.48.0.0/12',
         'KG': '158.181.128.0/17',
         'KH': '36.37.128.0/17',
         'KI': '103.25.140.0/22',
         'KM': '197.255.224.0/20',

-        'KN': '198.32.32.0/19',
+        'KN': '198.167.192.0/19',

         'KP': '175.45.176.0/22',
         'KR': '175.192.0.0/10',
         'KW': '37.36.0.0/14',
         'KP': '175.45.176.0/22',
         'KR': '175.192.0.0/10',
         'KW': '37.36.0.0/14',


@@ -5099,10 +5121,10 @@ class GeoUtils(object):
         'KZ': '2.72.0.0/13',
         'LA': '115.84.64.0/18',
         'LB': '178.135.0.0/16',
         'KZ': '2.72.0.0/13',
         'LA': '115.84.64.0/18',
         'LB': '178.135.0.0/16',

-        'LC': '192.147.231.0/24',
+        'LC': '24.92.144.0/20',

         'LI': '82.117.0.0/19',
         'LK': '112.134.0.0/15',
         'LI': '82.117.0.0/19',
         'LK': '112.134.0.0/15',

-        'LR': '41.86.0.0/19',
+        'LR': '102.183.0.0/16',

         'LS': '129.232.0.0/17',
         'LT': '78.56.0.0/13',
         'LU': '188.42.0.0/16',
         'LS': '129.232.0.0/17',
         'LT': '78.56.0.0/13',
         'LU': '188.42.0.0/16',


@@ -5127,7 +5149,7 @@ class GeoUtils(object):
         'MT': '46.11.0.0/16',
         'MU': '105.16.0.0/12',
         'MV': '27.114.128.0/18',
         'MT': '46.11.0.0/16',
         'MU': '105.16.0.0/12',
         'MV': '27.114.128.0/18',

-        'MW': '105.234.0.0/16',
+        'MW': '102.70.0.0/15',

         'MX': '187.192.0.0/11',
         'MY': '175.136.0.0/13',
         'MZ': '197.218.0.0/15',
         'MX': '187.192.0.0/11',
         'MY': '175.136.0.0/13',
         'MZ': '197.218.0.0/15',


@@ -5158,23 +5180,23 @@ class GeoUtils(object):
         'PW': '202.124.224.0/20',
         'PY': '181.120.0.0/14',
         'QA': '37.210.0.0/15',
         'PW': '202.124.224.0/20',
         'PY': '181.120.0.0/14',
         'QA': '37.210.0.0/15',

-        'RE': '139.26.0.0/16',
+        'RE': '102.35.0.0/16',

         'RO': '79.112.0.0/13',
         'RO': '79.112.0.0/13',

-        'RS': '178.220.0.0/14',
+        'RS': '93.86.0.0/15',

         'RU': '5.136.0.0/13',
         'RU': '5.136.0.0/13',

-        'RW': '105.178.0.0/15',
+        'RW': '41.186.0.0/16',

         'SA': '188.48.0.0/13',
         'SB': '202.1.160.0/19',
         'SC': '154.192.0.0/11',
         'SA': '188.48.0.0/13',
         'SB': '202.1.160.0/19',
         'SC': '154.192.0.0/11',

-        'SD': '154.96.0.0/13',
+        'SD': '102.120.0.0/13',

         'SE': '78.64.0.0/12',
         'SE': '78.64.0.0/12',

-        'SG': '152.56.0.0/14',
+        'SG': '8.128.0.0/10',

         'SI': '188.196.0.0/14',
         'SK': '78.98.0.0/15',
         'SI': '188.196.0.0/14',
         'SK': '78.98.0.0/15',

-        'SL': '197.215.0.0/17',
+        'SL': '102.143.0.0/17',

         'SM': '89.186.32.0/19',
         'SN': '41.82.0.0/15',
         'SM': '89.186.32.0/19',
         'SN': '41.82.0.0/15',

-        'SO': '197.220.64.0/19',
+        'SO': '154.115.192.0/18',

         'SR': '186.179.128.0/17',
         'SS': '105.235.208.0/21',
         'ST': '197.159.160.0/19',
         'SR': '186.179.128.0/17',
         'SS': '105.235.208.0/21',
         'ST': '197.159.160.0/19',


@@ -5197,15 +5219,15 @@ class GeoUtils(object):
         'TV': '202.2.96.0/19',
         'TW': '120.96.0.0/11',
         'TZ': '156.156.0.0/14',
         'TV': '202.2.96.0/19',
         'TW': '120.96.0.0/11',
         'TZ': '156.156.0.0/14',

-        'UA': '93.72.0.0/13',
-        'UG': '154.224.0.0/13',
-        'US': '3.0.0.0/8',
+        'UA': '37.52.0.0/14',
+        'UG': '102.80.0.0/13',
+        'US': '6.0.0.0/8',

         'UY': '167.56.0.0/13',
         'UY': '167.56.0.0/13',

-        'UZ': '82.215.64.0/18',
+        'UZ': '84.54.64.0/18',

         'VA': '212.77.0.0/19',
         'VA': '212.77.0.0/19',

-        'VC': '24.92.144.0/20',
+        'VC': '207.191.240.0/21',

         'VE': '186.88.0.0/13',
         'VE': '186.88.0.0/13',

-        'VG': '172.103.64.0/18',
+        'VG': '66.81.192.0/20',

         'VI': '146.226.0.0/16',
         'VN': '14.160.0.0/11',
         'VU': '202.80.32.0/20',
         'VI': '146.226.0.0/16',
         'VN': '14.160.0.0/11',
         'VU': '202.80.32.0/20',


@@ -5214,8 +5236,8 @@ class GeoUtils(object):
         'YE': '134.35.0.0/16',
         'YT': '41.242.116.0/22',
         'ZA': '41.0.0.0/11',
         'YE': '134.35.0.0/16',
         'YT': '41.242.116.0/22',
         'ZA': '41.0.0.0/11',

-        'ZM': '165.56.0.0/13',
-        'ZW': '41.85.192.0/19',
+        'ZM': '102.144.0.0/13',
+        'ZW': '102.177.192.0/18',

     }
 
     @classmethod
     }
 
     @classmethod


@@ -5377,6 +5399,19 @@ def decode_packed_codes(code):
         obfucasted_code)
 
 
         obfucasted_code)
 
 

+def caesar(s, alphabet, shift):
+    if shift == 0:
+        return s
+    l = len(alphabet)
+    return ''.join(
+        alphabet[(alphabet.index(c) + shift) % l] if c in alphabet else c
+        for c in s)
+
+
+def rot47(s):
+    return caesar(s, r'''!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''', 47)
+
+

 def parse_m3u8_attributes(attrib):
     info = {}
     for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
 def parse_m3u8_attributes(attrib):
     info = {}
     for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):