d/rules: Run offline tests.

[youtubedl] / youtube_dl / extractor / eighttracks.py

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index f21ef88530d2f8913b4b35d9c03fc4fc14de7ddc..9a44f89f3fe801047ef283e9b1fa410c8b88cb37 100644 (file)
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -1,8 +1,13 @@
+# coding: utf-8
+from __future__ import unicode_literals
+

 import json
 import random
 import json
 import random

-import re

 
 from .common import InfoExtractor
 
 from .common import InfoExtractor

+from ..compat import (
+    compat_str,
+)

 from ..utils import (
     ExtractorError,
 )
 from ..utils import (
     ExtractorError,
 )


@@ -10,110 +15,150 @@ from ..utils import (
 
 class EightTracksIE(InfoExtractor):
     IE_NAME = '8tracks'
 
 class EightTracksIE(InfoExtractor):
     IE_NAME = '8tracks'

-    _VALID_URL = r'https?://8tracks.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$'
+    _VALID_URL = r'https?://8tracks\.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$'

     _TEST = {
     _TEST = {

-        u"name": u"EightTracks",
-        u"url": u"http://8tracks.com/ytdl/youtube-dl-test-tracks-a",
-        u"playlist": [
+        'name': 'EightTracks',
+        'url': 'http://8tracks.com/ytdl/youtube-dl-test-tracks-a',
+        'info_dict': {
+            'id': '1336550',
+            'display_id': 'youtube-dl-test-tracks-a',
+            'description': "test chars:  \"'/\\ä↭",
+            'title': "youtube-dl test tracks \"'/\\ä↭<>",
+        },
+        'playlist': [

             {
             {

-                u"file": u"11885610.m4a",
-                u"md5": u"96ce57f24389fc8734ce47f4c1abcc55",
-                u"info_dict": {
-                    u"title": u"youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
-                    u"uploader_id": u"ytdl"
+                'md5': '96ce57f24389fc8734ce47f4c1abcc55',
+                'info_dict': {
+                    'id': '11885610',
+                    'ext': 'm4a',
+                    'title': "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'

                 }
             },
             {
                 }
             },
             {

-                u"file": u"11885608.m4a",
-                u"md5": u"4ab26f05c1f7291ea460a3920be8021f",
-                u"info_dict": {
-                    u"title": u"youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
-                    u"uploader_id": u"ytdl"
+                'md5': '4ab26f05c1f7291ea460a3920be8021f',
+                'info_dict': {
+                    'id': '11885608',
+                    'ext': 'm4a',
+                    'title': "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'

                 }
             },
             {
                 }
             },
             {

-                u"file": u"11885679.m4a",
-                u"md5": u"d30b5b5f74217410f4689605c35d1fd7",
-                u"info_dict": {
-                    u"title": u"youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
-                    u"uploader_id": u"ytdl"
+                'md5': 'd30b5b5f74217410f4689605c35d1fd7',
+                'info_dict': {
+                    'id': '11885679',
+                    'ext': 'm4a',
+                    'title': "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'

                 }
             },
             {
                 }
             },
             {

-                u"file": u"11885680.m4a",
-                u"md5": u"4eb0a669317cd725f6bbd336a29f923a",
-                u"info_dict": {
-                    u"title": u"youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
-                    u"uploader_id": u"ytdl"
+                'md5': '4eb0a669317cd725f6bbd336a29f923a',
+                'info_dict': {
+                    'id': '11885680',
+                    'ext': 'm4a',
+                    'title': "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'

                 }
             },
             {
                 }
             },
             {

-                u"file": u"11885682.m4a",
-                u"md5": u"1893e872e263a2705558d1d319ad19e8",
-                u"info_dict": {
-                    u"title": u"PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
-                    u"uploader_id": u"ytdl"
+                'md5': '1893e872e263a2705558d1d319ad19e8',
+                'info_dict': {
+                    'id': '11885682',
+                    'ext': 'm4a',
+                    'title': "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'

                 }
             },
             {
                 }
             },
             {

-                u"file": u"11885683.m4a",
-                u"md5": u"b673c46f47a216ab1741ae8836af5899",
-                u"info_dict": {
-                    u"title": u"PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
-                    u"uploader_id": u"ytdl"
+                'md5': 'b673c46f47a216ab1741ae8836af5899',
+                'info_dict': {
+                    'id': '11885683',
+                    'ext': 'm4a',
+                    'title': "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'

                 }
             },
             {
                 }
             },
             {

-                u"file": u"11885684.m4a",
-                u"md5": u"1d74534e95df54986da7f5abf7d842b7",
-                u"info_dict": {
-                    u"title": u"phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
-                    u"uploader_id": u"ytdl"
+                'md5': '1d74534e95df54986da7f5abf7d842b7',
+                'info_dict': {
+                    'id': '11885684',
+                    'ext': 'm4a',
+                    'title': "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'

                 }
             },
             {
                 }
             },
             {

-                u"file": u"11885685.m4a",
-                u"md5": u"f081f47af8f6ae782ed131d38b9cd1c0",
-                u"info_dict": {
-                    u"title": u"phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
-                    u"uploader_id": u"ytdl"
+                'md5': 'f081f47af8f6ae782ed131d38b9cd1c0',
+                'info_dict': {
+                    'id': '11885685',
+                    'ext': 'm4a',
+                    'title': "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
+                    'uploader_id': 'ytdl'

                 }
             }
         ]
     }
 
                 }
             }
         ]
     }
 

-

     def _real_extract(self, url):
     def _real_extract(self, url):

-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)

 
         webpage = self._download_webpage(url, playlist_id)
 
 
         webpage = self._download_webpage(url, playlist_id)
 

-        json_like = self._search_regex(r"PAGE.mix = (.*?);\n", webpage, u'trax information', flags=re.DOTALL)
-        data = json.loads(json_like)
+        data = self._parse_json(
+            self._search_regex(
+                r"(?s)PAGE\.mix\s*=\s*({.+?});\n", webpage, 'trax information'),
+            playlist_id)

 
         session = str(random.randint(0, 1000000000))
         mix_id = data['id']
         track_count = data['tracks_count']
 
         session = str(random.randint(0, 1000000000))
         mix_id = data['id']
         track_count = data['tracks_count']

+        duration = data['duration']
+        avg_song_duration = float(duration) / track_count
+        # duration is sometimes negative, use predefined avg duration
+        if avg_song_duration <= 0:
+            avg_song_duration = 300

         first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
         next_url = first_url
         first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
         next_url = first_url

-        res = []
+        entries = []
+

         for i in range(track_count):
         for i in range(track_count):

-            api_json = self._download_webpage(next_url, playlist_id,
-                note=u'Downloading song information %s/%s' % (str(i+1), track_count),
-                errnote=u'Failed to download song information')
+            api_json = None
+            download_tries = 0
+
+            while api_json is None:
+                try:
+                    api_json = self._download_webpage(
+                        next_url, playlist_id,
+                        note='Downloading song information %d/%d' % (i + 1, track_count),
+                        errnote='Failed to download song information')
+                except ExtractorError:
+                    if download_tries > 3:
+                        raise
+                    else:
+                        download_tries += 1
+                        self._sleep(avg_song_duration, playlist_id)
+

             api_data = json.loads(api_json)
             api_data = json.loads(api_json)

-            track_data = api_data[u'set']['track']
+            track_data = api_data['set']['track']

             info = {
             info = {

-                'id': track_data['id'],
+                'id': compat_str(track_data['id']),

                 'url': track_data['track_file_stream_url'],
                 'url': track_data['track_file_stream_url'],

-                'title': track_data['performer'] + u' - ' + track_data['name'],
+                'title': track_data['performer'] + ' - ' + track_data['name'],

                 'raw_title': track_data['name'],
                 'uploader_id': data['user']['login'],
                 'ext': 'm4a',
             }
                 'raw_title': track_data['name'],
                 'uploader_id': data['user']['login'],
                 'ext': 'm4a',
             }

-            res.append(info)
-            next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id'])
-        return res
+            entries.append(info)
+
+            next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (
+                session, mix_id, track_data['id'])
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': compat_str(mix_id),
+            'display_id': playlist_id,
+            'title': data.get('name'),
+            'description': data.get('description'),
+        }