X-Git-Url: https://git.rapsys.eu/youtubedl/blobdiff_plain/6d835d4d6903cf5e18ad01844736929e06d16004..56553d2fc728271c5fe6d6a3ecc9dfc16dacc35c:/test/test_utils.py diff --git a/test/test_utils.py b/test/test_utils.py index 0b31d1a..e13e11b 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -16,38 +16,52 @@ import json import xml.etree.ElementTree from youtube_dl.utils import ( + age_restricted, + args_to_str, + clean_html, DateRange, + detect_exe_version, encodeFilename, + escape_rfc3986, + escape_url, + ExtractorError, find_xpath_attr, fix_xml_ampersands, - get_meta_content, - orderedSet, - OnDemandPagedList, InAdvancePagedList, + intlist_to_bytes, + is_html, + js_to_json, + limit_length, + OnDemandPagedList, + orderedSet, parse_duration, + parse_filesize, + parse_iso8601, read_batch_urls, sanitize_filename, + sanitize_path, + prepend_extension, + replace_extension, shell_quote, smuggle_url, str_to_int, + strip_jsonp, struct_unpack, timeconvert, unescapeHTML, unified_strdate, unsmuggle_url, + uppercase_escape, + lowercase_escape, url_basename, urlencode_postdata, + version_tuple, xpath_with_ns, - parse_iso8601, - strip_jsonp, - uppercase_escape, - limit_length, - escape_rfc3986, - escape_url, - js_to_json, - get_filesystem_encoding, - compat_getenv, - compat_expanduser, + xpath_text, + render_table, + match_str, + parse_dfxp_time_expr, + dfxp2srt, ) @@ -76,6 +90,15 @@ class TestUtil(unittest.TestCase): tests = '\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430' self.assertEqual(sanitize_filename(tests), tests) + self.assertEqual( + sanitize_filename('New World record at 0:12:34'), + 'New World record at 0_12_34') + + self.assertEqual(sanitize_filename('--gasdgf'), '_-gasdgf') + self.assertEqual(sanitize_filename('--gasdgf', is_id=True), '--gasdgf') + self.assertEqual(sanitize_filename('.gasdgf'), 'gasdgf') + self.assertEqual(sanitize_filename('.gasdgf', is_id=True), '.gasdgf') + forbidden = '"\0\\/' for fc in forbidden: for fbc in forbidden: @@ -116,20 +139,74 @@ class TestUtil(unittest.TestCase): self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw') self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI') + def test_sanitize_path(self): + if sys.platform != 'win32': + return + + self.assertEqual(sanitize_path('abc'), 'abc') + self.assertEqual(sanitize_path('abc/def'), 'abc\\def') + self.assertEqual(sanitize_path('abc\\def'), 'abc\\def') + self.assertEqual(sanitize_path('abc|def'), 'abc#def') + self.assertEqual(sanitize_path('<>:"|?*'), '#######') + self.assertEqual(sanitize_path('C:/abc/def'), 'C:\\abc\\def') + self.assertEqual(sanitize_path('C?:/abc/def'), 'C##\\abc\\def') + + self.assertEqual(sanitize_path('\\\\?\\UNC\\ComputerName\\abc'), '\\\\?\\UNC\\ComputerName\\abc') + self.assertEqual(sanitize_path('\\\\?\\UNC/ComputerName/abc'), '\\\\?\\UNC\\ComputerName\\abc') + + self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc') + self.assertEqual(sanitize_path('\\\\?\\C:/abc'), '\\\\?\\C:\\abc') + self.assertEqual(sanitize_path('\\\\?\\C:\\ab?c\\de:f'), '\\\\?\\C:\\ab#c\\de#f') + self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc') + + self.assertEqual( + sanitize_path('youtube/%(uploader)s/%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s'), + 'youtube\\%(uploader)s\\%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s') + + self.assertEqual( + sanitize_path('youtube/TheWreckingYard ./00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part'), + 'youtube\\TheWreckingYard #\\00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part') + self.assertEqual(sanitize_path('abc/def...'), 'abc\\def..#') + self.assertEqual(sanitize_path('abc.../def'), 'abc..#\\def') + self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#') + + self.assertEqual(sanitize_path('../abc'), '..\\abc') + self.assertEqual(sanitize_path('../../abc'), '..\\..\\abc') + self.assertEqual(sanitize_path('./abc'), 'abc') + self.assertEqual(sanitize_path('./../abc'), '..\\abc') + + def test_prepend_extension(self): + self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext') + self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext') + self.assertEqual(prepend_extension('abc.unexpected_ext', 'temp', 'ext'), 'abc.unexpected_ext.temp') + self.assertEqual(prepend_extension('abc', 'temp'), 'abc.temp') + self.assertEqual(prepend_extension('.abc', 'temp'), '.abc.temp') + self.assertEqual(prepend_extension('.abc.ext', 'temp'), '.abc.temp.ext') + + def test_replace_extension(self): + self.assertEqual(replace_extension('abc.ext', 'temp'), 'abc.temp') + self.assertEqual(replace_extension('abc.ext', 'temp', 'ext'), 'abc.temp') + self.assertEqual(replace_extension('abc.unexpected_ext', 'temp', 'ext'), 'abc.unexpected_ext.temp') + self.assertEqual(replace_extension('abc', 'temp'), 'abc.temp') + self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp') + self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp') + def test_ordered_set(self): self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7]) self.assertEqual(orderedSet([]), []) self.assertEqual(orderedSet([1]), [1]) - #keep the list ordered + # keep the list ordered self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1]) def test_unescape_html(self): self.assertEqual(unescapeHTML('%20;'), '%20;') + self.assertEqual(unescapeHTML('/'), '/') + self.assertEqual(unescapeHTML('/'), '/') self.assertEqual( unescapeHTML('é'), 'é') - + def test_daterange(self): - _20century = DateRange("19000101","20000101") + _20century = DateRange("19000101", "20000101") self.assertFalse("17890714" in _20century) _ac = DateRange("00010101") self.assertTrue("19690721" in _ac) @@ -141,8 +218,16 @@ class TestUtil(unittest.TestCase): self.assertEqual(unified_strdate('8/7/2009'), '20090708') self.assertEqual(unified_strdate('Dec 14, 2012'), '20121214') self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011') + self.assertEqual(unified_strdate('1968 12 10'), '19681210') self.assertEqual(unified_strdate('1968-12-10'), '19681210') self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128') + self.assertEqual( + unified_strdate('11/26/2014 11:30:00 AM PST', day_first=False), + '20141126') + self.assertEqual( + unified_strdate('2/2/2015 6:47:40 PM', day_first=False), + '20150202') + self.assertEqual(unified_strdate('25-09-2014'), '20140925') def test_find_xpath_attr(self): testxml = ''' @@ -157,17 +242,6 @@ class TestUtil(unittest.TestCase): self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1]) self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2]) - def test_meta_parser(self): - testhtml = ''' - - - - - ''' - get_meta = lambda name: get_meta_content(name, testhtml) - self.assertEqual(get_meta('description'), 'foo & bar') - self.assertEqual(get_meta('author'), 'Plato') - def test_xpath_with_ns(self): testxml = ''' @@ -181,8 +255,19 @@ class TestUtil(unittest.TestCase): self.assertEqual(find('media:song/media:author').text, 'The Author') self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3') + def test_xpath_text(self): + testxml = ''' +
+

Foo

+
+ ''' + doc = xml.etree.ElementTree.fromstring(testxml) + self.assertEqual(xpath_text(doc, 'div/p'), 'Foo') + self.assertTrue(xpath_text(doc, 'div/bar') is None) + self.assertRaises(ExtractorError, xpath_text, doc, 'div/bar', fatal=True) + def test_smuggle_url(self): - data = {u"ö": u"ö", u"abc": [3]} + data = {"ö": "ö", "abc": [3]} url = 'https://foo.bar/baz?x=y#a' smug_url = smuggle_url(url, data) unsmug_url, unsmug_data = unsmuggle_url(smug_url) @@ -213,6 +298,8 @@ class TestUtil(unittest.TestCase): def test_parse_duration(self): self.assertEqual(parse_duration(None), None) + self.assertEqual(parse_duration(False), None) + self.assertEqual(parse_duration('invalid'), None) self.assertEqual(parse_duration('1'), 1) self.assertEqual(parse_duration('1337:12'), 80232) self.assertEqual(parse_duration('9:12:43'), 33163) @@ -230,6 +317,13 @@ class TestUtil(unittest.TestCase): self.assertEqual(parse_duration('0m0s'), 0) self.assertEqual(parse_duration('0s'), 0) self.assertEqual(parse_duration('01:02:03.05'), 3723.05) + self.assertEqual(parse_duration('T30M38S'), 1838) + self.assertEqual(parse_duration('5 s'), 5) + self.assertEqual(parse_duration('3 min'), 180) + self.assertEqual(parse_duration('2.5 hours'), 9000) + self.assertEqual(parse_duration('02:03:04'), 7384) + self.assertEqual(parse_duration('01:02:03:04'), 93784) + self.assertEqual(parse_duration('1 hour 3 minutes'), 3780) def test_fix_xml_ampersands(self): self.assertEqual( @@ -296,10 +390,18 @@ class TestUtil(unittest.TestCase): d = json.loads(stripped) self.assertEqual(d, [{"id": "532cb", "x": 3}]) + stripped = strip_jsonp('parseMetadata({"STATUS":"OK"})\n\n\n//epc') + d = json.loads(stripped) + self.assertEqual(d, {'STATUS': 'OK'}) + def test_uppercase_escape(self): self.assertEqual(uppercase_escape('aä'), 'aä') self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐') + def test_lowercase_escape(self): + self.assertEqual(lowercase_escape('aä'), 'aä') + self.assertEqual(lowercase_escape('\\u0026'), '&') + def test_limit_length(self): self.assertEqual(limit_length(None, 12), None) self.assertEqual(limit_length('foo', 12), 'foo') @@ -352,6 +454,10 @@ class TestUtil(unittest.TestCase): "playlist":[{"controls":{"all":null}}] }''') + inp = '"SAND Number: SAND 2013-7800P\\nPresenter: Tom Russo\\nHabanero Software Training - Xyce Software\\nXyce, Sandia\\u0027s"' + json_code = js_to_json(inp) + self.assertEqual(json.loads(json_code), json.loads(inp)) + def test_js_to_json_edgecases(self): on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}") self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"}) @@ -359,17 +465,177 @@ class TestUtil(unittest.TestCase): on = js_to_json('{"abc": true}') self.assertEqual(json.loads(on), {'abc': True}) - def test_compat_getenv(self): - test_str = 'тест' - os.environ['YOUTUBE-DL-TEST'] = (test_str if sys.version_info >= (3, 0) - else test_str.encode(get_filesystem_encoding())) - self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str) - - def test_compat_expanduser(self): - test_str = 'C:\Documents and Settings\тест\Application Data' - os.environ['HOME'] = (test_str if sys.version_info >= (3, 0) - else test_str.encode(get_filesystem_encoding())) - self.assertEqual(compat_expanduser('~'), test_str) + # Ignore JavaScript code as well + on = js_to_json('''{ + "x": 1, + y: "a", + z: some.code + }''') + d = json.loads(on) + self.assertEqual(d['x'], 1) + self.assertEqual(d['y'], 'a') + + on = js_to_json('["abc", "def",]') + self.assertEqual(json.loads(on), ['abc', 'def']) + + on = js_to_json('{"abc": "def",}') + self.assertEqual(json.loads(on), {'abc': 'def'}) + + def test_clean_html(self): + self.assertEqual(clean_html('a:\nb'), 'a: b') + self.assertEqual(clean_html('a:\n "b"'), 'a: "b"') + + def test_intlist_to_bytes(self): + self.assertEqual( + intlist_to_bytes([0, 1, 127, 128, 255]), + b'\x00\x01\x7f\x80\xff') + + def test_args_to_str(self): + self.assertEqual( + args_to_str(['foo', 'ba/r', '-baz', '2 be', '']), + 'foo ba/r -baz \'2 be\' \'\'' + ) + + def test_parse_filesize(self): + self.assertEqual(parse_filesize(None), None) + self.assertEqual(parse_filesize(''), None) + self.assertEqual(parse_filesize('91 B'), 91) + self.assertEqual(parse_filesize('foobar'), None) + self.assertEqual(parse_filesize('2 MiB'), 2097152) + self.assertEqual(parse_filesize('5 GB'), 5000000000) + self.assertEqual(parse_filesize('1.2Tb'), 1200000000000) + self.assertEqual(parse_filesize('1,24 KB'), 1240) + + def test_version_tuple(self): + self.assertEqual(version_tuple('1'), (1,)) + self.assertEqual(version_tuple('10.23.344'), (10, 23, 344)) + self.assertEqual(version_tuple('10.1-6'), (10, 1, 6)) # avconv style + + def test_detect_exe_version(self): + self.assertEqual(detect_exe_version('''ffmpeg version 1.2.1 +built on May 27 2013 08:37:26 with gcc 4.7 (Debian 4.7.3-4) +configuration: --prefix=/usr --extra-'''), '1.2.1') + self.assertEqual(detect_exe_version('''ffmpeg version N-63176-g1fb4685 +built on May 15 2014 22:09:06 with gcc 4.8.2 (GCC)'''), 'N-63176-g1fb4685') + self.assertEqual(detect_exe_version('''X server found. dri2 connection failed! +Trying to open render node... +Success at /dev/dri/renderD128. +ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4') + + def test_age_restricted(self): + self.assertFalse(age_restricted(None, 10)) # unrestricted content + self.assertFalse(age_restricted(1, None)) # unrestricted policy + self.assertFalse(age_restricted(8, 10)) + self.assertTrue(age_restricted(18, 14)) + self.assertFalse(age_restricted(18, 18)) + + def test_is_html(self): + self.assertFalse(is_html(b'\x49\x44\x43\xaaa')) + self.assertTrue(is_html( # UTF-8 with BOM + b'\xef\xbb\xbf\xaaa')) + self.assertTrue(is_html( # UTF-16-LE + b'\xff\xfe<\x00h\x00t\x00m\x00l\x00>\x00\xe4\x00' + )) + self.assertTrue(is_html( # UTF-16-BE + b'\xfe\xff\x00<\x00h\x00t\x00m\x00l\x00>\x00\xe4' + )) + self.assertTrue(is_html( # UTF-32-BE + b'\x00\x00\xFE\xFF\x00\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4')) + self.assertTrue(is_html( # UTF-32-LE + b'\xFF\xFE\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4\x00\x00\x00')) + + def test_render_table(self): + self.assertEqual( + render_table( + ['a', 'bcd'], + [[123, 4], [9999, 51]]), + 'a bcd\n' + '123 4\n' + '9999 51') + + def test_match_str(self): + self.assertRaises(ValueError, match_str, 'xy>foobar', {}) + self.assertFalse(match_str('xy', {'x': 1200})) + self.assertTrue(match_str('!xy', {'x': 1200})) + self.assertTrue(match_str('x', {'x': 1200})) + self.assertFalse(match_str('!x', {'x': 1200})) + self.assertTrue(match_str('x', {'x': 0})) + self.assertFalse(match_str('x>0', {'x': 0})) + self.assertFalse(match_str('x>0', {})) + self.assertTrue(match_str('x>?0', {})) + self.assertTrue(match_str('x>1K', {'x': 1200})) + self.assertFalse(match_str('x>2K', {'x': 1200})) + self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200})) + self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200})) + self.assertFalse(match_str('y=a212', {'y': 'foobar42'})) + self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'})) + self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'})) + self.assertTrue(match_str('y!=foobar2', {'y': 'foobar42'})) + self.assertFalse(match_str( + 'like_count > 100 & dislike_count 100 & dislike_count 100 & dislike_count 100 & dislike_count + + +
+

The following line contains Chinese characters and special symbols

+

第二行
♪♪

+

Third
Line

+
+ + ''' + srt_data = '''1 +00:00:00,000 --> 00:00:01,000 +The following line contains Chinese characters and special symbols + +2 +00:00:01,000 --> 00:00:02,000 +第二行 +♪♪ + +3 +00:00:02,000 --> 00:00:03,000 +Third +Line + +''' + self.assertEqual(dfxp2srt(dfxp_data), srt_data) + + dfxp_data_no_default_namespace = ''' + + +
+

The first line

+
+ + ''' + srt_data = '''1 +00:00:00,000 --> 00:00:01,000 +The first line + +''' + self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data) + if __name__ == '__main__': unittest.main()