debian/control: Update list of supported sites.

[youtubedl] / youtube_dl / utils.py

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 38262bee4cd08f97e4f5009d7066d1466bde25a4..d1eca3760a66e89fbfc1673e70a2d526035776cf 100644 (file)
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -7,6 +7,7 @@ import base64
 import binascii
 import calendar
 import codecs
 import binascii
 import calendar
 import codecs

+import collections

 import contextlib
 import ctypes
 import datetime
 import contextlib
 import ctypes
 import datetime


@@ -30,6 +31,7 @@ import ssl
 import subprocess
 import sys
 import tempfile
 import subprocess
 import sys
 import tempfile

+import time

 import traceback
 import xml.etree.ElementTree
 import zlib
 import traceback
 import xml.etree.ElementTree
 import zlib


@@ -1835,6 +1837,12 @@ def write_json_file(obj, fn):
                 os.unlink(fn)
             except OSError:
                 pass
                 os.unlink(fn)
             except OSError:
                 pass

+        try:
+            mask = os.umask(0)
+            os.umask(mask)
+            os.chmod(tf.name, 0o666 & ~mask)
+        except OSError:
+            pass

         os.rename(tf.name, fn)
     except Exception:
         try:
         os.rename(tf.name, fn)
     except Exception:
         try:


@@ -2735,14 +2743,66 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
     1. https://curl.haxx.se/docs/http-cookies.html
     """
     _HTTPONLY_PREFIX = '#HttpOnly_'
     1. https://curl.haxx.se/docs/http-cookies.html
     """
     _HTTPONLY_PREFIX = '#HttpOnly_'

+    _ENTRY_LEN = 7
+    _HEADER = '''# Netscape HTTP Cookie File
+# This file is generated by youtube-dl.  Do not edit.
+
+'''
+    _CookieFileEntry = collections.namedtuple(
+        'CookieFileEntry',
+        ('domain_name', 'include_subdomains', 'path', 'https_only', 'expires_at', 'name', 'value'))

 
     def save(self, filename=None, ignore_discard=False, ignore_expires=False):
 
     def save(self, filename=None, ignore_discard=False, ignore_expires=False):

+        """
+        Save cookies to a file.
+
+        Most of the code is taken from CPython 3.8 and slightly adapted
+        to support cookie files with UTF-8 in both python 2 and 3.
+        """
+        if filename is None:
+            if self.filename is not None:
+                filename = self.filename
+            else:
+                raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
+

         # Store session cookies with `expires` set to 0 instead of an empty
         # string
         for cookie in self:
             if cookie.expires is None:
                 cookie.expires = 0
         # Store session cookies with `expires` set to 0 instead of an empty
         # string
         for cookie in self:
             if cookie.expires is None:
                 cookie.expires = 0

-        compat_cookiejar.MozillaCookieJar.save(self, filename, ignore_discard, ignore_expires)
+
+        with io.open(filename, 'w', encoding='utf-8') as f:
+            f.write(self._HEADER)
+            now = time.time()
+            for cookie in self:
+                if not ignore_discard and cookie.discard:
+                    continue
+                if not ignore_expires and cookie.is_expired(now):
+                    continue
+                if cookie.secure:
+                    secure = 'TRUE'
+                else:
+                    secure = 'FALSE'
+                if cookie.domain.startswith('.'):
+                    initial_dot = 'TRUE'
+                else:
+                    initial_dot = 'FALSE'
+                if cookie.expires is not None:
+                    expires = compat_str(cookie.expires)
+                else:
+                    expires = ''
+                if cookie.value is None:
+                    # cookies.txt regards 'Set-Cookie: foo' as a cookie
+                    # with no name, whereas http.cookiejar regards it as a
+                    # cookie with no value.
+                    name = ''
+                    value = cookie.name
+                else:
+                    name = cookie.name
+                    value = cookie.value
+                f.write(
+                    '\t'.join([cookie.domain, initial_dot, cookie.path,
+                               secure, expires, name, value]) + '\n')

 
     def load(self, filename=None, ignore_discard=False, ignore_expires=False):
         """Load cookies from a file."""
 
     def load(self, filename=None, ignore_discard=False, ignore_expires=False):
         """Load cookies from a file."""


@@ -2752,12 +2812,30 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
             else:
                 raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
 
             else:
                 raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
 

+        def prepare_line(line):
+            if line.startswith(self._HTTPONLY_PREFIX):
+                line = line[len(self._HTTPONLY_PREFIX):]
+            # comments and empty lines are fine
+            if line.startswith('#') or not line.strip():
+                return line
+            cookie_list = line.split('\t')
+            if len(cookie_list) != self._ENTRY_LEN:
+                raise compat_cookiejar.LoadError('invalid length %d' % len(cookie_list))
+            cookie = self._CookieFileEntry(*cookie_list)
+            if cookie.expires_at and not cookie.expires_at.isdigit():
+                raise compat_cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
+            return line
+

         cf = io.StringIO()
         cf = io.StringIO()

-        with open(filename) as f:
+        with io.open(filename, encoding='utf-8') as f:

             for line in f:
             for line in f:

-                if line.startswith(self._HTTPONLY_PREFIX):
-                    line = line[len(self._HTTPONLY_PREFIX):]
-                cf.write(compat_str(line))
+                try:
+                    cf.write(prepare_line(line))
+                except compat_cookiejar.LoadError as e:
+                    write_string(
+                        'WARNING: skipping cookie file entry due to %s: %r\n'
+                        % (e, line), sys.stderr)
+                    continue

         cf.seek(0)
         self._really_load(cf, filename, ignore_discard, ignore_expires)
         # Session cookies are denoted by either `expires` field set to
         cf.seek(0)
         self._really_load(cf, filename, ignore_discard, ignore_expires)
         # Session cookies are denoted by either `expires` field set to