X-Git-Url: https://git.rapsys.eu/youtubedl/blobdiff_plain/feb5020b37d7d3ba4005a8bac6f4efece4ce4b8c..a5a28b1eabe0533f26d23c4157c97144215b22b7:/youtube_dl/extractor/eighttracks.py diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py index cced068..9a44f89 100644 --- a/youtube_dl/extractor/eighttracks.py +++ b/youtube_dl/extractor/eighttracks.py @@ -1,9 +1,13 @@ -import itertools +# coding: utf-8 +from __future__ import unicode_literals + import json import random -import re from .common import InfoExtractor +from ..compat import ( + compat_str, +) from ..utils import ( ExtractorError, ) @@ -11,112 +15,150 @@ from ..utils import ( class EightTracksIE(InfoExtractor): IE_NAME = '8tracks' - _VALID_URL = r'https?://8tracks.com/(?P[^/]+)/(?P[^/#]+)(?:#.*)?$' + _VALID_URL = r'https?://8tracks\.com/(?P[^/]+)/(?P[^/#]+)(?:#.*)?$' _TEST = { - u"name": u"EightTracks", - u"url": u"http://8tracks.com/ytdl/youtube-dl-test-tracks-a", - u"playlist": [ + 'name': 'EightTracks', + 'url': 'http://8tracks.com/ytdl/youtube-dl-test-tracks-a', + 'info_dict': { + 'id': '1336550', + 'display_id': 'youtube-dl-test-tracks-a', + 'description': "test chars: \"'/\\ä↭", + 'title': "youtube-dl test tracks \"'/\\ä↭<>", + }, + 'playlist': [ { - u"file": u"11885610.m4a", - u"md5": u"96ce57f24389fc8734ce47f4c1abcc55", - u"info_dict": { - u"title": u"youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad", - u"uploader_id": u"ytdl" + 'md5': '96ce57f24389fc8734ce47f4c1abcc55', + 'info_dict': { + 'id': '11885610', + 'ext': 'm4a', + 'title': "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad", + 'uploader_id': 'ytdl' } }, { - u"file": u"11885608.m4a", - u"md5": u"4ab26f05c1f7291ea460a3920be8021f", - u"info_dict": { - u"title": u"youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad", - u"uploader_id": u"ytdl" + 'md5': '4ab26f05c1f7291ea460a3920be8021f', + 'info_dict': { + 'id': '11885608', + 'ext': 'm4a', + 'title': "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad", + 'uploader_id': 'ytdl' } }, { - u"file": u"11885679.m4a", - u"md5": u"d30b5b5f74217410f4689605c35d1fd7", - u"info_dict": { - u"title": u"youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad", - u"uploader_id": u"ytdl" + 'md5': 'd30b5b5f74217410f4689605c35d1fd7', + 'info_dict': { + 'id': '11885679', + 'ext': 'm4a', + 'title': "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad", + 'uploader_id': 'ytdl' } }, { - u"file": u"11885680.m4a", - u"md5": u"4eb0a669317cd725f6bbd336a29f923a", - u"info_dict": { - u"title": u"youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad", - u"uploader_id": u"ytdl" + 'md5': '4eb0a669317cd725f6bbd336a29f923a', + 'info_dict': { + 'id': '11885680', + 'ext': 'm4a', + 'title': "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad", + 'uploader_id': 'ytdl' } }, { - u"file": u"11885682.m4a", - u"md5": u"1893e872e263a2705558d1d319ad19e8", - u"info_dict": { - u"title": u"PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad", - u"uploader_id": u"ytdl" + 'md5': '1893e872e263a2705558d1d319ad19e8', + 'info_dict': { + 'id': '11885682', + 'ext': 'm4a', + 'title': "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad", + 'uploader_id': 'ytdl' } }, { - u"file": u"11885683.m4a", - u"md5": u"b673c46f47a216ab1741ae8836af5899", - u"info_dict": { - u"title": u"PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad", - u"uploader_id": u"ytdl" + 'md5': 'b673c46f47a216ab1741ae8836af5899', + 'info_dict': { + 'id': '11885683', + 'ext': 'm4a', + 'title': "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad", + 'uploader_id': 'ytdl' } }, { - u"file": u"11885684.m4a", - u"md5": u"1d74534e95df54986da7f5abf7d842b7", - u"info_dict": { - u"title": u"phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad", - u"uploader_id": u"ytdl" + 'md5': '1d74534e95df54986da7f5abf7d842b7', + 'info_dict': { + 'id': '11885684', + 'ext': 'm4a', + 'title': "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad", + 'uploader_id': 'ytdl' } }, { - u"file": u"11885685.m4a", - u"md5": u"f081f47af8f6ae782ed131d38b9cd1c0", - u"info_dict": { - u"title": u"phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad", - u"uploader_id": u"ytdl" + 'md5': 'f081f47af8f6ae782ed131d38b9cd1c0', + 'info_dict': { + 'id': '11885685', + 'ext': 'm4a', + 'title': "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad", + 'uploader_id': 'ytdl' } } ] } - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - playlist_id = mobj.group('id') + playlist_id = self._match_id(url) webpage = self._download_webpage(url, playlist_id) - json_like = self._search_regex(r"PAGE.mix = (.*?);\n", webpage, u'trax information', flags=re.DOTALL) - data = json.loads(json_like) + data = self._parse_json( + self._search_regex( + r"(?s)PAGE\.mix\s*=\s*({.+?});\n", webpage, 'trax information'), + playlist_id) session = str(random.randint(0, 1000000000)) mix_id = data['id'] track_count = data['tracks_count'] + duration = data['duration'] + avg_song_duration = float(duration) / track_count + # duration is sometimes negative, use predefined avg duration + if avg_song_duration <= 0: + avg_song_duration = 300 first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id) next_url = first_url - res = [] - for i in itertools.count(): - api_json = self._download_webpage(next_url, playlist_id, - note=u'Downloading song information %s/%s' % (str(i+1), track_count), - errnote=u'Failed to download song information') + entries = [] + + for i in range(track_count): + api_json = None + download_tries = 0 + + while api_json is None: + try: + api_json = self._download_webpage( + next_url, playlist_id, + note='Downloading song information %d/%d' % (i + 1, track_count), + errnote='Failed to download song information') + except ExtractorError: + if download_tries > 3: + raise + else: + download_tries += 1 + self._sleep(avg_song_duration, playlist_id) + api_data = json.loads(api_json) - track_data = api_data[u'set']['track'] + track_data = api_data['set']['track'] info = { - 'id': track_data['id'], + 'id': compat_str(track_data['id']), 'url': track_data['track_file_stream_url'], - 'title': track_data['performer'] + u' - ' + track_data['name'], + 'title': track_data['performer'] + ' - ' + track_data['name'], 'raw_title': track_data['name'], 'uploader_id': data['user']['login'], 'ext': 'm4a', } - res.append(info) - if api_data['set']['at_last_track']: - break - next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id']) - return res + entries.append(info) + + next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % ( + session, mix_id, track_data['id']) + return { + '_type': 'playlist', + 'entries': entries, + 'id': compat_str(mix_id), + 'display_id': playlist_id, + 'title': data.get('name'), + 'description': data.get('description'), + }