X-Git-Url: https://git.rapsys.eu/youtubedl/blobdiff_plain/46113edab215c2211a604c06245c16d5d4e57dcf..540a20889afafc522db30a941b5cb2f97eb885ff:/test/test_download.py diff --git a/test/test_download.py b/test/test_download.py index 75e0bb2..ebe820d 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +from __future__ import unicode_literals + # Allow direct execution import os import sys @@ -23,10 +25,12 @@ import json import socket import youtube_dl.YoutubeDL -from youtube_dl.utils import ( +from youtube_dl.compat import ( compat_http_client, compat_urllib_error, compat_HTTPError, +) +from youtube_dl.utils import ( DownloadError, ExtractorError, format_bytes, @@ -36,36 +40,60 @@ from youtube_dl.extractor import get_info_extractor RETRIES = 3 + class YoutubeDL(youtube_dl.YoutubeDL): def __init__(self, *args, **kwargs): self.to_stderr = self.to_screen self.processed_info_dicts = [] super(YoutubeDL, self).__init__(*args, **kwargs) + def report_warning(self, message): # Don't accept warnings during tests raise ExtractorError(message) + def process_info(self, info_dict): self.processed_info_dicts.append(info_dict) return super(YoutubeDL, self).process_info(info_dict) + def _file_md5(fn): with open(fn, 'rb') as f: return hashlib.md5(f.read()).hexdigest() + defs = gettestcases() class TestDownload(unittest.TestCase): + # Parallel testing in nosetests. See + # http://nose.readthedocs.org/en/latest/doc_tests/test_multiprocess/multiprocess.html + _multiprocess_shared_ = True + maxDiff = None + + def __str__(self): + """Identify each test with the `add_ie` attribute, if available.""" + + def strclass(cls): + """From 2.7's unittest; 2.6 had _strclass so we can't import it.""" + return '%s.%s' % (cls.__module__, cls.__name__) + + add_ie = getattr(self, self._testMethodName).add_ie + return '%s (%s)%s:' % (self._testMethodName, + strclass(self.__class__), + ' [%s]' % add_ie if add_ie else '') + def setUp(self): self.defs = defs -### Dynamically generate tests -def generator(test_case): +# Dynamically generate tests + + +def generator(test_case, tname): def test_template(self): - ie = youtube_dl.extractor.get_info_extractor(test_case['name']) - other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])] + ie = youtube_dl.extractor.get_info_extractor(test_case['name'])() + other_ies = [get_info_extractor(ie_key)() for ie_key in test_case.get('add_ie', [])] is_playlist = any(k.startswith('playlist') for k in test_case) test_cases = test_case.get( 'playlist', [] if is_playlist else [test_case]) @@ -78,7 +106,7 @@ def generator(test_case): for tc in test_cases: info_dict = tc.get('info_dict', {}) - if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')): + if not (info_dict.get('id') and info_dict.get('ext')): raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?') if 'skip' in test_case: @@ -86,17 +114,19 @@ def generator(test_case): return for other_ie in other_ies: if not other_ie.working(): - print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key()) + print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key()) return params = get_params(test_case.get('params', {})) + params['outtmpl'] = tname + '_' + params['outtmpl'] if is_playlist and 'playlist' not in test_case: - params.setdefault('extract_flat', True) + params.setdefault('extract_flat', 'in_playlist') params.setdefault('skip_download', True) - ydl = YoutubeDL(params) + ydl = YoutubeDL(params, auto_init=False) ydl.add_default_info_extractors() finished_hook_called = set() + def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) @@ -104,9 +134,10 @@ def generator(test_case): expect_warnings(ydl, test_case.get('expected_warnings', [])) def get_tc_filename(tc): - return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) + return ydl.prepare_filename(tc.get('info_dict', {})) res_dict = None + def try_rm_tcs_files(tcs=None): if tcs is None: tcs = test_cases @@ -120,17 +151,19 @@ def generator(test_case): try_num = 1 while True: try: - # We're not using .download here sine that is just a shim + # We're not using .download here since that is just a shim # for outside error handling, and returns the exit code # instead of the result dict. - res_dict = ydl.extract_info(test_case['url']) + res_dict = ydl.extract_info( + test_case['url'], + force_generic_extractor=params.get('force_generic_extractor', False)) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): raise if try_num == RETRIES: - report_warning(u'Failed due to network errors, skipping...') + report_warning('%s failed due to network errors, skipping...' % tname) return print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num)) @@ -140,9 +173,9 @@ def generator(test_case): break if is_playlist: - self.assertEqual(res_dict['_type'], 'playlist') + self.assertTrue(res_dict['_type'] in ['playlist', 'multi_video']) self.assertTrue('entries' in res_dict) - expect_info_dict(self, test_case.get('info_dict', {}), res_dict) + expect_info_dict(self, res_dict, test_case.get('info_dict', {})) if 'playlist_mincount' in test_case: assertGreaterEqual( @@ -166,7 +199,16 @@ def generator(test_case): self.assertEqual( test_case['playlist_duration_sum'], got_duration) - for tc in test_cases: + # Generalize both playlists and single videos to unified format for + # simplicity + if 'entries' not in res_dict: + res_dict['entries'] = [res_dict] + + for tc_num, tc in enumerate(test_cases): + tc_res_dict = res_dict['entries'][tc_num] + # First, check test cases' data against extracted data alone + expect_info_dict(self, tc_res_dict, tc.get('info_dict', {})) + # Now, check downloaded file consistency tc_filename = get_tc_filename(tc) if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename) @@ -183,15 +225,16 @@ def generator(test_case): format_bytes(got_fsize))) if 'md5' in tc: md5_for_file = _file_md5(tc_filename) - self.assertEqual(md5_for_file, tc['md5']) + self.assertEqual(tc['md5'], md5_for_file) + # Finally, check test cases' data again but this time against + # extracted data from info JSON file written during processing info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json' self.assertTrue( os.path.exists(info_json_fn), 'Missing info file %s' % info_json_fn) with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) - - expect_info_dict(self, tc.get('info_dict', {}), info_dict) + expect_info_dict(self, info_dict, tc.get('info_dict', {})) finally: try_rm_tcs_files() if is_playlist and res_dict is not None and res_dict.get('entries'): @@ -202,15 +245,18 @@ def generator(test_case): return test_template -### And add them to TestDownload + +# And add them to TestDownload for n, test_case in enumerate(defs): - test_method = generator(test_case) tname = 'test_' + str(test_case['name']) i = 1 while hasattr(TestDownload, tname): - tname = 'test_' + str(test_case['name']) + '_' + str(i) + tname = 'test_%s_%d' % (test_case['name'], i) i += 1 - test_method.__name__ = tname + test_method = generator(test_case, tname) + test_method.__name__ = str(tname) + ie_list = test_case.get('add_ie') + test_method.add_ie = ie_list and ','.join(ie_list) setattr(TestDownload, test_method.__name__, test_method) del test_method