]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/mixcloud.py
Prepare to release.
[youtubedl] / youtube_dl / extractor / mixcloud.py
1 from __future__ import unicode_literals
2
3 import base64
4 import functools
5 import itertools
6 import re
7
8 from .common import InfoExtractor
9 from ..compat import (
10 compat_chr,
11 compat_ord,
12 compat_urllib_parse_unquote,
13 compat_urlparse,
14 )
15 from ..utils import (
16 clean_html,
17 ExtractorError,
18 OnDemandPagedList,
19 str_to_int,
20 )
21
22
23 class MixcloudIE(InfoExtractor):
24 _VALID_URL = r'https?://(?:(?:www|beta|m)\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
25 IE_NAME = 'mixcloud'
26
27 _TESTS = [{
28 'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
29 'info_dict': {
30 'id': 'dholbach-cryptkeeper',
31 'ext': 'm4a',
32 'title': 'Cryptkeeper',
33 'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
34 'uploader': 'Daniel Holbach',
35 'uploader_id': 'dholbach',
36 'thumbnail': r're:https?://.*\.jpg',
37 'view_count': int,
38 },
39 }, {
40 'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
41 'info_dict': {
42 'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat',
43 'ext': 'mp3',
44 'title': 'Caribou 7 inch Vinyl Mix & Chat',
45 'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
46 'uploader': 'Gilles Peterson Worldwide',
47 'uploader_id': 'gillespeterson',
48 'thumbnail': 're:https?://.*',
49 'view_count': int,
50 },
51 }, {
52 'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
53 'only_matching': True,
54 }]
55
56 # See https://www.mixcloud.com/media/js2/www_js_2.9e23256562c080482435196ca3975ab5.js
57 @staticmethod
58 def _decrypt_play_info(play_info):
59 KEY = 'pleasedontdownloadourmusictheartistswontgetpaid'
60
61 play_info = base64.b64decode(play_info.encode('ascii'))
62
63 return ''.join([
64 compat_chr(compat_ord(ch) ^ compat_ord(KEY[idx % len(KEY)]))
65 for idx, ch in enumerate(play_info)])
66
67 def _real_extract(self, url):
68 mobj = re.match(self._VALID_URL, url)
69 uploader = mobj.group(1)
70 cloudcast_name = mobj.group(2)
71 track_id = compat_urllib_parse_unquote('-'.join((uploader, cloudcast_name)))
72
73 webpage = self._download_webpage(url, track_id)
74
75 message = self._html_search_regex(
76 r'(?s)<div[^>]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)',
77 webpage, 'error message', default=None)
78
79 encrypted_play_info = self._search_regex(
80 r'm-play-info="([^"]+)"', webpage, 'play info')
81 play_info = self._parse_json(
82 self._decrypt_play_info(encrypted_play_info), track_id)
83
84 if message and 'stream_url' not in play_info:
85 raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
86
87 song_url = play_info['stream_url']
88
89 title = self._html_search_regex(r'm-title="([^"]+)"', webpage, 'title')
90 thumbnail = self._proto_relative_url(self._html_search_regex(
91 r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail', fatal=False))
92 uploader = self._html_search_regex(
93 r'm-owner-name="([^"]+)"', webpage, 'uploader', fatal=False)
94 uploader_id = self._search_regex(
95 r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
96 description = self._og_search_description(webpage)
97 view_count = str_to_int(self._search_regex(
98 [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
99 r'/listeners/?">([0-9,.]+)</a>',
100 r'm-tooltip=["\']([\d,.]+) plays'],
101 webpage, 'play count', default=None))
102
103 return {
104 'id': track_id,
105 'title': title,
106 'url': song_url,
107 'description': description,
108 'thumbnail': thumbnail,
109 'uploader': uploader,
110 'uploader_id': uploader_id,
111 'view_count': view_count,
112 }
113
114
115 class MixcloudPlaylistBaseIE(InfoExtractor):
116 _PAGE_SIZE = 24
117
118 def _find_urls_in_page(self, page):
119 for url in re.findall(r'm-play-button m-url="(?P<url>[^"]+)"', page):
120 yield self.url_result(
121 compat_urlparse.urljoin('https://www.mixcloud.com', clean_html(url)),
122 MixcloudIE.ie_key())
123
124 def _fetch_tracks_page(self, path, video_id, page_name, current_page, real_page_number=None):
125 real_page_number = real_page_number or current_page + 1
126 return self._download_webpage(
127 'https://www.mixcloud.com/%s/' % path, video_id,
128 note='Download %s (page %d)' % (page_name, current_page + 1),
129 errnote='Unable to download %s' % page_name,
130 query={'page': real_page_number, 'list': 'main', '_ajax': '1'},
131 headers={'X-Requested-With': 'XMLHttpRequest'})
132
133 def _tracks_page_func(self, page, video_id, page_name, current_page):
134 resp = self._fetch_tracks_page(page, video_id, page_name, current_page)
135
136 for item in self._find_urls_in_page(resp):
137 yield item
138
139 def _get_user_description(self, page_content):
140 return self._html_search_regex(
141 r'<div[^>]+class="description-text"[^>]*>(.+?)</div>',
142 page_content, 'user description', fatal=False)
143
144
145 class MixcloudUserIE(MixcloudPlaylistBaseIE):
146 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/(?P<type>uploads|favorites|listens)?/?$'
147 IE_NAME = 'mixcloud:user'
148
149 _TESTS = [{
150 'url': 'http://www.mixcloud.com/dholbach/',
151 'info_dict': {
152 'id': 'dholbach_uploads',
153 'title': 'Daniel Holbach (uploads)',
154 'description': 'md5:327af72d1efeb404a8216c27240d1370',
155 },
156 'playlist_mincount': 11,
157 }, {
158 'url': 'http://www.mixcloud.com/dholbach/uploads/',
159 'info_dict': {
160 'id': 'dholbach_uploads',
161 'title': 'Daniel Holbach (uploads)',
162 'description': 'md5:327af72d1efeb404a8216c27240d1370',
163 },
164 'playlist_mincount': 11,
165 }, {
166 'url': 'http://www.mixcloud.com/dholbach/favorites/',
167 'info_dict': {
168 'id': 'dholbach_favorites',
169 'title': 'Daniel Holbach (favorites)',
170 'description': 'md5:327af72d1efeb404a8216c27240d1370',
171 },
172 'params': {
173 'playlist_items': '1-100',
174 },
175 'playlist_mincount': 100,
176 }, {
177 'url': 'http://www.mixcloud.com/dholbach/listens/',
178 'info_dict': {
179 'id': 'dholbach_listens',
180 'title': 'Daniel Holbach (listens)',
181 'description': 'md5:327af72d1efeb404a8216c27240d1370',
182 },
183 'params': {
184 'playlist_items': '1-100',
185 },
186 'playlist_mincount': 100,
187 }]
188
189 def _real_extract(self, url):
190 mobj = re.match(self._VALID_URL, url)
191 user_id = mobj.group('user')
192 list_type = mobj.group('type')
193
194 # if only a profile URL was supplied, default to download all uploads
195 if list_type is None:
196 list_type = 'uploads'
197
198 video_id = '%s_%s' % (user_id, list_type)
199
200 profile = self._download_webpage(
201 'https://www.mixcloud.com/%s/' % user_id, video_id,
202 note='Downloading user profile',
203 errnote='Unable to download user profile')
204
205 username = self._og_search_title(profile)
206 description = self._get_user_description(profile)
207
208 entries = OnDemandPagedList(
209 functools.partial(
210 self._tracks_page_func,
211 '%s/%s' % (user_id, list_type), video_id, 'list of %s' % list_type),
212 self._PAGE_SIZE, use_cache=True)
213
214 return self.playlist_result(
215 entries, video_id, '%s (%s)' % (username, list_type), description)
216
217
218 class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
219 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
220 IE_NAME = 'mixcloud:playlist'
221
222 _TESTS = [{
223 'url': 'https://www.mixcloud.com/RedBullThre3style/playlists/tokyo-finalists-2015/',
224 'info_dict': {
225 'id': 'RedBullThre3style_tokyo-finalists-2015',
226 'title': 'National Champions 2015',
227 'description': 'md5:6ff5fb01ac76a31abc9b3939c16243a3',
228 },
229 'playlist_mincount': 16,
230 }, {
231 'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
232 'info_dict': {
233 'id': 'maxvibes_jazzcat-on-ness-radio',
234 'title': 'Jazzcat on Ness Radio',
235 'description': 'md5:7bbbf0d6359a0b8cda85224be0f8f263',
236 },
237 'playlist_mincount': 23
238 }]
239
240 def _real_extract(self, url):
241 mobj = re.match(self._VALID_URL, url)
242 user_id = mobj.group('user')
243 playlist_id = mobj.group('playlist')
244 video_id = '%s_%s' % (user_id, playlist_id)
245
246 profile = self._download_webpage(
247 url, user_id,
248 note='Downloading playlist page',
249 errnote='Unable to download playlist page')
250
251 description = self._get_user_description(profile)
252 playlist_title = self._html_search_regex(
253 r'<span[^>]+class="[^"]*list-playlist-title[^"]*"[^>]*>(.*?)</span>',
254 profile, 'playlist title')
255
256 entries = OnDemandPagedList(
257 functools.partial(
258 self._tracks_page_func,
259 '%s/playlists/%s' % (user_id, playlist_id), video_id, 'tracklist'),
260 self._PAGE_SIZE)
261
262 return self.playlist_result(entries, video_id, playlist_title, description)
263
264
265 class MixcloudStreamIE(MixcloudPlaylistBaseIE):
266 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/stream/?$'
267 IE_NAME = 'mixcloud:stream'
268
269 _TEST = {
270 'url': 'https://www.mixcloud.com/FirstEar/stream/',
271 'info_dict': {
272 'id': 'FirstEar',
273 'title': 'First Ear',
274 'description': 'Curators of good music\nfirstearmusic.com',
275 },
276 'playlist_mincount': 192,
277 }
278
279 def _real_extract(self, url):
280 user_id = self._match_id(url)
281
282 webpage = self._download_webpage(url, user_id)
283
284 entries = []
285 prev_page_url = None
286
287 def _handle_page(page):
288 entries.extend(self._find_urls_in_page(page))
289 return self._search_regex(
290 r'm-next-page-url="([^"]+)"', page,
291 'next page URL', default=None)
292
293 next_page_url = _handle_page(webpage)
294
295 for idx in itertools.count(0):
296 if not next_page_url or prev_page_url == next_page_url:
297 break
298
299 prev_page_url = next_page_url
300 current_page = int(self._search_regex(
301 r'\?page=(\d+)', next_page_url, 'next page number'))
302
303 next_page_url = _handle_page(self._fetch_tracks_page(
304 '%s/stream' % user_id, user_id, 'stream', idx,
305 real_page_number=current_page))
306
307 username = self._og_search_title(webpage)
308 description = self._get_user_description(webpage)
309
310 return self.playlist_result(entries, user_id, username, description)