]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/mixcloud.py
560fe188b675a619785332eea285484fa85154bf
[youtubedl] / youtube_dl / extractor / mixcloud.py
1 from __future__ import unicode_literals
2
3 import base64
4 import functools
5 import itertools
6 import re
7
8 from .common import InfoExtractor
9 from ..compat import (
10 compat_chr,
11 compat_ord,
12 compat_urllib_parse_unquote,
13 compat_urlparse,
14 )
15 from ..utils import (
16 clean_html,
17 ExtractorError,
18 OnDemandPagedList,
19 parse_count,
20 str_to_int,
21 )
22
23
24 class MixcloudIE(InfoExtractor):
25 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
26 IE_NAME = 'mixcloud'
27
28 _TESTS = [{
29 'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
30 'info_dict': {
31 'id': 'dholbach-cryptkeeper',
32 'ext': 'm4a',
33 'title': 'Cryptkeeper',
34 'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
35 'uploader': 'Daniel Holbach',
36 'uploader_id': 'dholbach',
37 'thumbnail': 're:https?://.*\.jpg',
38 'view_count': int,
39 'like_count': int,
40 },
41 }, {
42 'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
43 'info_dict': {
44 'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat',
45 'ext': 'mp3',
46 'title': 'Caribou 7 inch Vinyl Mix & Chat',
47 'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
48 'uploader': 'Gilles Peterson Worldwide',
49 'uploader_id': 'gillespeterson',
50 'thumbnail': 're:https?://.*',
51 'view_count': int,
52 'like_count': int,
53 },
54 }]
55
56 # See https://www.mixcloud.com/media/js2/www_js_2.9e23256562c080482435196ca3975ab5.js
57 @staticmethod
58 def _decrypt_play_info(play_info):
59 KEY = 'pleasedontdownloadourmusictheartistswontgetpaid'
60
61 play_info = base64.b64decode(play_info.encode('ascii'))
62
63 return ''.join([
64 compat_chr(compat_ord(ch) ^ compat_ord(KEY[idx % len(KEY)]))
65 for idx, ch in enumerate(play_info)])
66
67 def _real_extract(self, url):
68 mobj = re.match(self._VALID_URL, url)
69 uploader = mobj.group(1)
70 cloudcast_name = mobj.group(2)
71 track_id = compat_urllib_parse_unquote('-'.join((uploader, cloudcast_name)))
72
73 webpage = self._download_webpage(url, track_id)
74
75 message = self._html_search_regex(
76 r'(?s)<div[^>]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)',
77 webpage, 'error message', default=None)
78
79 encrypted_play_info = self._search_regex(
80 r'm-play-info="([^"]+)"', webpage, 'play info')
81 play_info = self._parse_json(
82 self._decrypt_play_info(encrypted_play_info), track_id)
83
84 if message and 'stream_url' not in play_info:
85 raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
86
87 song_url = play_info['stream_url']
88
89 PREFIX = (
90 r'm-play-on-spacebar[^>]+'
91 r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
92 title = self._html_search_regex(
93 PREFIX + r'm-title="([^"]+)"', webpage, 'title')
94 thumbnail = self._proto_relative_url(self._html_search_regex(
95 PREFIX + r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail',
96 fatal=False))
97 uploader = self._html_search_regex(
98 PREFIX + r'm-owner-name="([^"]+)"',
99 webpage, 'uploader', fatal=False)
100 uploader_id = self._search_regex(
101 r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
102 description = self._og_search_description(webpage)
103 like_count = parse_count(self._search_regex(
104 r'\bbutton-favorite[^>]+>.*?<span[^>]+class=["\']toggle-number[^>]+>\s*([^<]+)',
105 webpage, 'like count', default=None))
106 view_count = str_to_int(self._search_regex(
107 [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
108 r'/listeners/?">([0-9,.]+)</a>'],
109 webpage, 'play count', default=None))
110
111 return {
112 'id': track_id,
113 'title': title,
114 'url': song_url,
115 'description': description,
116 'thumbnail': thumbnail,
117 'uploader': uploader,
118 'uploader_id': uploader_id,
119 'view_count': view_count,
120 'like_count': like_count,
121 }
122
123
124 class MixcloudPlaylistBaseIE(InfoExtractor):
125 _PAGE_SIZE = 24
126
127 def _find_urls_in_page(self, page):
128 for url in re.findall(r'm-play-button m-url="(?P<url>[^"]+)"', page):
129 yield self.url_result(
130 compat_urlparse.urljoin('https://www.mixcloud.com', clean_html(url)),
131 MixcloudIE.ie_key())
132
133 def _fetch_tracks_page(self, path, video_id, page_name, current_page, real_page_number=None):
134 real_page_number = real_page_number or current_page + 1
135 return self._download_webpage(
136 'https://www.mixcloud.com/%s/' % path, video_id,
137 note='Download %s (page %d)' % (page_name, current_page + 1),
138 errnote='Unable to download %s' % page_name,
139 query={'page': real_page_number, 'list': 'main', '_ajax': '1'},
140 headers={'X-Requested-With': 'XMLHttpRequest'})
141
142 def _tracks_page_func(self, page, video_id, page_name, current_page):
143 resp = self._fetch_tracks_page(page, video_id, page_name, current_page)
144
145 for item in self._find_urls_in_page(resp):
146 yield item
147
148 def _get_user_description(self, page_content):
149 return self._html_search_regex(
150 r'<div[^>]+class="description-text"[^>]*>(.+?)</div>',
151 page_content, 'user description', fatal=False)
152
153
154 class MixcloudUserIE(MixcloudPlaylistBaseIE):
155 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/(?P<type>uploads|favorites|listens)?/?$'
156 IE_NAME = 'mixcloud:user'
157
158 _TESTS = [{
159 'url': 'http://www.mixcloud.com/dholbach/',
160 'info_dict': {
161 'id': 'dholbach_uploads',
162 'title': 'Daniel Holbach (uploads)',
163 'description': 'md5:327af72d1efeb404a8216c27240d1370',
164 },
165 'playlist_mincount': 11,
166 }, {
167 'url': 'http://www.mixcloud.com/dholbach/uploads/',
168 'info_dict': {
169 'id': 'dholbach_uploads',
170 'title': 'Daniel Holbach (uploads)',
171 'description': 'md5:327af72d1efeb404a8216c27240d1370',
172 },
173 'playlist_mincount': 11,
174 }, {
175 'url': 'http://www.mixcloud.com/dholbach/favorites/',
176 'info_dict': {
177 'id': 'dholbach_favorites',
178 'title': 'Daniel Holbach (favorites)',
179 'description': 'md5:327af72d1efeb404a8216c27240d1370',
180 },
181 'params': {
182 'playlist_items': '1-100',
183 },
184 'playlist_mincount': 100,
185 }, {
186 'url': 'http://www.mixcloud.com/dholbach/listens/',
187 'info_dict': {
188 'id': 'dholbach_listens',
189 'title': 'Daniel Holbach (listens)',
190 'description': 'md5:327af72d1efeb404a8216c27240d1370',
191 },
192 'params': {
193 'playlist_items': '1-100',
194 },
195 'playlist_mincount': 100,
196 }]
197
198 def _real_extract(self, url):
199 mobj = re.match(self._VALID_URL, url)
200 user_id = mobj.group('user')
201 list_type = mobj.group('type')
202
203 # if only a profile URL was supplied, default to download all uploads
204 if list_type is None:
205 list_type = 'uploads'
206
207 video_id = '%s_%s' % (user_id, list_type)
208
209 profile = self._download_webpage(
210 'https://www.mixcloud.com/%s/' % user_id, video_id,
211 note='Downloading user profile',
212 errnote='Unable to download user profile')
213
214 username = self._og_search_title(profile)
215 description = self._get_user_description(profile)
216
217 entries = OnDemandPagedList(
218 functools.partial(
219 self._tracks_page_func,
220 '%s/%s' % (user_id, list_type), video_id, 'list of %s' % list_type),
221 self._PAGE_SIZE, use_cache=True)
222
223 return self.playlist_result(
224 entries, video_id, '%s (%s)' % (username, list_type), description)
225
226
227 class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
228 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
229 IE_NAME = 'mixcloud:playlist'
230
231 _TESTS = [{
232 'url': 'https://www.mixcloud.com/RedBullThre3style/playlists/tokyo-finalists-2015/',
233 'info_dict': {
234 'id': 'RedBullThre3style_tokyo-finalists-2015',
235 'title': 'National Champions 2015',
236 'description': 'md5:6ff5fb01ac76a31abc9b3939c16243a3',
237 },
238 'playlist_mincount': 16,
239 }, {
240 'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
241 'info_dict': {
242 'id': 'maxvibes_jazzcat-on-ness-radio',
243 'title': 'Jazzcat on Ness Radio',
244 'description': 'md5:7bbbf0d6359a0b8cda85224be0f8f263',
245 },
246 'playlist_mincount': 23
247 }]
248
249 def _real_extract(self, url):
250 mobj = re.match(self._VALID_URL, url)
251 user_id = mobj.group('user')
252 playlist_id = mobj.group('playlist')
253 video_id = '%s_%s' % (user_id, playlist_id)
254
255 profile = self._download_webpage(
256 url, user_id,
257 note='Downloading playlist page',
258 errnote='Unable to download playlist page')
259
260 description = self._get_user_description(profile)
261 playlist_title = self._html_search_regex(
262 r'<span[^>]+class="[^"]*list-playlist-title[^"]*"[^>]*>(.*?)</span>',
263 profile, 'playlist title')
264
265 entries = OnDemandPagedList(
266 functools.partial(
267 self._tracks_page_func,
268 '%s/playlists/%s' % (user_id, playlist_id), video_id, 'tracklist'),
269 self._PAGE_SIZE)
270
271 return self.playlist_result(entries, video_id, playlist_title, description)
272
273
274 class MixcloudStreamIE(MixcloudPlaylistBaseIE):
275 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/stream/?$'
276 IE_NAME = 'mixcloud:stream'
277
278 _TEST = {
279 'url': 'https://www.mixcloud.com/FirstEar/stream/',
280 'info_dict': {
281 'id': 'FirstEar',
282 'title': 'First Ear',
283 'description': 'Curators of good music\nfirstearmusic.com',
284 },
285 'playlist_mincount': 192,
286 }
287
288 def _real_extract(self, url):
289 user_id = self._match_id(url)
290
291 webpage = self._download_webpage(url, user_id)
292
293 entries = []
294 prev_page_url = None
295
296 def _handle_page(page):
297 entries.extend(self._find_urls_in_page(page))
298 return self._search_regex(
299 r'm-next-page-url="([^"]+)"', page,
300 'next page URL', default=None)
301
302 next_page_url = _handle_page(webpage)
303
304 for idx in itertools.count(0):
305 if not next_page_url or prev_page_url == next_page_url:
306 break
307
308 prev_page_url = next_page_url
309 current_page = int(self._search_regex(
310 r'\?page=(\d+)', next_page_url, 'next page number'))
311
312 next_page_url = _handle_page(self._fetch_tracks_page(
313 '%s/stream' % user_id, user_id, 'stream', idx,
314 real_page_number=current_page))
315
316 username = self._og_search_title(webpage)
317 description = self._get_user_description(webpage)
318
319 return self.playlist_result(entries, user_id, username, description)