]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/soundcloud.py
d/p/skip-style-checks-for-flake8: Add metadata to please lintian.
[youtubedl] / youtube_dl / extractor / soundcloud.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import itertools
5 import re
6
7 from .common import (
8 InfoExtractor,
9 SearchInfoExtractor
10 )
11 from ..compat import (
12 compat_HTTPError,
13 compat_kwargs,
14 compat_str,
15 compat_urlparse,
16 )
17 from ..utils import (
18 error_to_compat_str,
19 ExtractorError,
20 float_or_none,
21 HEADRequest,
22 int_or_none,
23 KNOWN_EXTENSIONS,
24 mimetype2ext,
25 str_or_none,
26 try_get,
27 unified_timestamp,
28 update_url_query,
29 url_or_none,
30 )
31
32
33 class SoundcloudEmbedIE(InfoExtractor):
34 _VALID_URL = r'https?://(?:w|player|p)\.soundcloud\.com/player/?.*?\burl=(?P<id>.+)'
35 _TEST = {
36 # from https://www.soundi.fi/uutiset/ennakkokuuntelussa-timo-kaukolammen-station-to-station-to-station-julkaisua-juhlitaan-tanaan-g-livelabissa/
37 'url': 'https://w.soundcloud.com/player/?visual=true&url=https%3A%2F%2Fapi.soundcloud.com%2Fplaylists%2F922213810&show_artwork=true&maxwidth=640&maxheight=960&dnt=1&secret_token=s-ziYey',
38 'only_matching': True,
39 }
40
41 @staticmethod
42 def _extract_urls(webpage):
43 return [m.group('url') for m in re.finditer(
44 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:w\.)?soundcloud\.com/player.+?)\1',
45 webpage)]
46
47 def _real_extract(self, url):
48 query = compat_urlparse.parse_qs(
49 compat_urlparse.urlparse(url).query)
50 api_url = query['url'][0]
51 secret_token = query.get('secret_token')
52 if secret_token:
53 api_url = update_url_query(api_url, {'secret_token': secret_token[0]})
54 return self.url_result(api_url)
55
56
57 class SoundcloudIE(InfoExtractor):
58 """Information extractor for soundcloud.com
59 To access the media, the uid of the song and a stream token
60 must be extracted from the page source and the script must make
61 a request to media.soundcloud.com/crossdomain.xml. Then
62 the media can be grabbed by requesting from an url composed
63 of the stream token and uid
64 """
65
66 _VALID_URL = r'''(?x)^(?:https?://)?
67 (?:(?:(?:www\.|m\.)?soundcloud\.com/
68 (?!stations/track)
69 (?P<uploader>[\w\d-]+)/
70 (?!(?:tracks|albums|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
71 (?P<title>[\w\d-]+)/?
72 (?P<token>[^?]+?)?(?:[?].*)?$)
73 |(?:api(?:-v2)?\.soundcloud\.com/tracks/(?P<track_id>\d+)
74 (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
75 )
76 '''
77 IE_NAME = 'soundcloud'
78 _TESTS = [
79 {
80 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
81 'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
82 'info_dict': {
83 'id': '62986583',
84 'ext': 'mp3',
85 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
86 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
87 'uploader': 'E.T. ExTerrestrial Music',
88 'uploader_id': '1571244',
89 'timestamp': 1349920598,
90 'upload_date': '20121011',
91 'duration': 143.216,
92 'license': 'all-rights-reserved',
93 'view_count': int,
94 'like_count': int,
95 'comment_count': int,
96 'repost_count': int,
97 }
98 },
99 # not streamable song, preview
100 {
101 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
102 'info_dict': {
103 'id': '47127627',
104 'ext': 'mp3',
105 'title': 'Goldrushed',
106 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
107 'uploader': 'The Royal Concept',
108 'uploader_id': '9615865',
109 'timestamp': 1337635207,
110 'upload_date': '20120521',
111 'duration': 30,
112 'license': 'all-rights-reserved',
113 'view_count': int,
114 'like_count': int,
115 'comment_count': int,
116 'repost_count': int,
117 },
118 'params': {
119 # rtmp
120 'skip_download': True,
121 },
122 },
123 # private link
124 {
125 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
126 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
127 'info_dict': {
128 'id': '123998367',
129 'ext': 'mp3',
130 'title': 'Youtube - Dl Test Video \'\' Ä↭',
131 'description': 'test chars: \"\'/\\ä↭',
132 'uploader': 'jaimeMF',
133 'uploader_id': '69767071',
134 'timestamp': 1386604920,
135 'upload_date': '20131209',
136 'duration': 9.927,
137 'license': 'all-rights-reserved',
138 'view_count': int,
139 'like_count': int,
140 'comment_count': int,
141 'repost_count': int,
142 },
143 },
144 # private link (alt format)
145 {
146 'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp',
147 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
148 'info_dict': {
149 'id': '123998367',
150 'ext': 'mp3',
151 'title': 'Youtube - Dl Test Video \'\' Ä↭',
152 'description': 'test chars: \"\'/\\ä↭',
153 'uploader': 'jaimeMF',
154 'uploader_id': '69767071',
155 'timestamp': 1386604920,
156 'upload_date': '20131209',
157 'duration': 9.927,
158 'license': 'all-rights-reserved',
159 'view_count': int,
160 'like_count': int,
161 'comment_count': int,
162 'repost_count': int,
163 },
164 },
165 # downloadable song
166 {
167 'url': 'https://soundcloud.com/oddsamples/bus-brakes',
168 'md5': '7624f2351f8a3b2e7cd51522496e7631',
169 'info_dict': {
170 'id': '128590877',
171 'ext': 'mp3',
172 'title': 'Bus Brakes',
173 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
174 'uploader': 'oddsamples',
175 'uploader_id': '73680509',
176 'timestamp': 1389232924,
177 'upload_date': '20140109',
178 'duration': 17.346,
179 'license': 'cc-by-sa',
180 'view_count': int,
181 'like_count': int,
182 'comment_count': int,
183 'repost_count': int,
184 },
185 },
186 # private link, downloadable format
187 {
188 'url': 'https://soundcloud.com/oriuplift/uponly-238-no-talking-wav/s-AyZUd',
189 'md5': '64a60b16e617d41d0bef032b7f55441e',
190 'info_dict': {
191 'id': '340344461',
192 'ext': 'wav',
193 'title': 'Uplifting Only 238 [No Talking] (incl. Alex Feed Guestmix) (Aug 31, 2017) [wav]',
194 'description': 'md5:fa20ee0fca76a3d6df8c7e57f3715366',
195 'uploader': 'Ori Uplift Music',
196 'uploader_id': '12563093',
197 'timestamp': 1504206263,
198 'upload_date': '20170831',
199 'duration': 7449.096,
200 'license': 'all-rights-reserved',
201 'view_count': int,
202 'like_count': int,
203 'comment_count': int,
204 'repost_count': int,
205 },
206 },
207 # no album art, use avatar pic for thumbnail
208 {
209 'url': 'https://soundcloud.com/garyvee/sideways-prod-mad-real',
210 'md5': '59c7872bc44e5d99b7211891664760c2',
211 'info_dict': {
212 'id': '309699954',
213 'ext': 'mp3',
214 'title': 'Sideways (Prod. Mad Real)',
215 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
216 'uploader': 'garyvee',
217 'uploader_id': '2366352',
218 'timestamp': 1488152409,
219 'upload_date': '20170226',
220 'duration': 207.012,
221 'thumbnail': r're:https?://.*\.jpg',
222 'license': 'all-rights-reserved',
223 'view_count': int,
224 'like_count': int,
225 'comment_count': int,
226 'repost_count': int,
227 },
228 'params': {
229 'skip_download': True,
230 },
231 },
232 # not available via api.soundcloud.com/i1/tracks/id/streams
233 {
234 'url': 'https://soundcloud.com/giovannisarani/mezzo-valzer',
235 'md5': 'e22aecd2bc88e0e4e432d7dcc0a1abf7',
236 'info_dict': {
237 'id': '583011102',
238 'ext': 'mp3',
239 'title': 'Mezzo Valzer',
240 'description': 'md5:4138d582f81866a530317bae316e8b61',
241 'uploader': 'Giovanni Sarani',
242 'uploader_id': '3352531',
243 'timestamp': 1551394171,
244 'upload_date': '20190228',
245 'duration': 180.157,
246 'thumbnail': r're:https?://.*\.jpg',
247 'license': 'all-rights-reserved',
248 'view_count': int,
249 'like_count': int,
250 'comment_count': int,
251 'repost_count': int,
252 },
253 'expected_warnings': ['Unable to download JSON metadata'],
254 }
255 ]
256
257 _API_BASE = 'https://api.soundcloud.com/'
258 _API_V2_BASE = 'https://api-v2.soundcloud.com/'
259 _BASE_URL = 'https://soundcloud.com/'
260 _IMAGE_REPL_RE = r'-([0-9a-z]+)\.jpg'
261
262 _ARTWORK_MAP = {
263 'mini': 16,
264 'tiny': 20,
265 'small': 32,
266 'badge': 47,
267 't67x67': 67,
268 'large': 100,
269 't300x300': 300,
270 'crop': 400,
271 't500x500': 500,
272 'original': 0,
273 }
274
275 def _store_client_id(self, client_id):
276 self._downloader.cache.store('soundcloud', 'client_id', client_id)
277
278 def _update_client_id(self):
279 webpage = self._download_webpage('https://soundcloud.com/', None)
280 for src in reversed(re.findall(r'<script[^>]+src="([^"]+)"', webpage)):
281 script = self._download_webpage(src, None, fatal=False)
282 if script:
283 client_id = self._search_regex(
284 r'client_id\s*:\s*"([0-9a-zA-Z]{32})"',
285 script, 'client id', default=None)
286 if client_id:
287 self._CLIENT_ID = client_id
288 self._store_client_id(client_id)
289 return
290 raise ExtractorError('Unable to extract client id')
291
292 def _download_json(self, *args, **kwargs):
293 non_fatal = kwargs.get('fatal') is False
294 if non_fatal:
295 del kwargs['fatal']
296 query = kwargs.get('query', {}).copy()
297 for _ in range(2):
298 query['client_id'] = self._CLIENT_ID
299 kwargs['query'] = query
300 try:
301 return super(SoundcloudIE, self)._download_json(*args, **compat_kwargs(kwargs))
302 except ExtractorError as e:
303 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
304 self._store_client_id(None)
305 self._update_client_id()
306 continue
307 elif non_fatal:
308 self._downloader.report_warning(error_to_compat_str(e))
309 return False
310 raise
311
312 def _real_initialize(self):
313 self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk'
314
315 @classmethod
316 def _resolv_url(cls, url):
317 return SoundcloudIE._API_V2_BASE + 'resolve?url=' + url
318
319 def _extract_info_dict(self, info, full_title=None, secret_token=None, version=2):
320 track_id = compat_str(info['id'])
321 title = info['title']
322 track_base_url = self._API_BASE + 'tracks/%s' % track_id
323
324 format_urls = set()
325 formats = []
326 query = {'client_id': self._CLIENT_ID}
327 if secret_token:
328 query['secret_token'] = secret_token
329
330 if info.get('downloadable') and info.get('has_downloads_left'):
331 format_url = update_url_query(
332 info.get('download_url') or track_base_url + '/download', query)
333 format_urls.add(format_url)
334 if version == 2:
335 v1_info = self._download_json(
336 track_base_url, track_id, query=query, fatal=False) or {}
337 else:
338 v1_info = info
339 formats.append({
340 'format_id': 'download',
341 'ext': v1_info.get('original_format') or 'mp3',
342 'filesize': int_or_none(v1_info.get('original_content_size')),
343 'url': format_url,
344 'preference': 10,
345 })
346
347 def invalid_url(url):
348 return not url or url in format_urls
349
350 def add_format(f, protocol, is_preview=False):
351 mobj = re.search(r'\.(?P<abr>\d+)\.(?P<ext>[0-9a-z]{3,4})(?=[/?])', stream_url)
352 if mobj:
353 for k, v in mobj.groupdict().items():
354 if not f.get(k):
355 f[k] = v
356 format_id_list = []
357 if protocol:
358 format_id_list.append(protocol)
359 for k in ('ext', 'abr'):
360 v = f.get(k)
361 if v:
362 format_id_list.append(v)
363 preview = is_preview or re.search(r'/(?:preview|playlist)/0/30/', f['url'])
364 if preview:
365 format_id_list.append('preview')
366 abr = f.get('abr')
367 if abr:
368 f['abr'] = int(abr)
369 f.update({
370 'format_id': '_'.join(format_id_list),
371 'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
372 'preference': -10 if preview else None,
373 })
374 formats.append(f)
375
376 # New API
377 transcodings = try_get(
378 info, lambda x: x['media']['transcodings'], list) or []
379 for t in transcodings:
380 if not isinstance(t, dict):
381 continue
382 format_url = url_or_none(t.get('url'))
383 if not format_url:
384 continue
385 stream = self._download_json(
386 format_url, track_id, query=query, fatal=False)
387 if not isinstance(stream, dict):
388 continue
389 stream_url = url_or_none(stream.get('url'))
390 if invalid_url(stream_url):
391 continue
392 format_urls.add(stream_url)
393 stream_format = t.get('format') or {}
394 protocol = stream_format.get('protocol')
395 if protocol != 'hls' and '/hls' in format_url:
396 protocol = 'hls'
397 ext = None
398 preset = str_or_none(t.get('preset'))
399 if preset:
400 ext = preset.split('_')[0]
401 if ext not in KNOWN_EXTENSIONS:
402 ext = mimetype2ext(stream_format.get('mime_type'))
403 add_format({
404 'url': stream_url,
405 'ext': ext,
406 }, 'http' if protocol == 'progressive' else protocol,
407 t.get('snipped') or '/preview/' in format_url)
408
409 if not formats:
410 # Old API, does not work for some tracks (e.g.
411 # https://soundcloud.com/giovannisarani/mezzo-valzer)
412 # and might serve preview URLs (e.g.
413 # http://www.soundcloud.com/snbrn/ele)
414 format_dict = self._download_json(
415 track_base_url + '/streams', track_id,
416 'Downloading track url', query=query, fatal=False) or {}
417
418 for key, stream_url in format_dict.items():
419 if invalid_url(stream_url):
420 continue
421 format_urls.add(stream_url)
422 mobj = re.search(r'(http|hls)_([^_]+)_(\d+)_url', key)
423 if mobj:
424 protocol, ext, abr = mobj.groups()
425 add_format({
426 'abr': abr,
427 'ext': ext,
428 'url': stream_url,
429 }, protocol)
430
431 if not formats:
432 # We fallback to the stream_url in the original info, this
433 # cannot be always used, sometimes it can give an HTTP 404 error
434 urlh = self._request_webpage(
435 HEADRequest(info.get('stream_url') or track_base_url + '/stream'),
436 track_id, query=query, fatal=False)
437 if urlh:
438 stream_url = urlh.geturl()
439 if not invalid_url(stream_url):
440 add_format({'url': stream_url}, 'http')
441
442 for f in formats:
443 f['vcodec'] = 'none'
444
445 self._sort_formats(formats)
446
447 user = info.get('user') or {}
448
449 thumbnails = []
450 artwork_url = info.get('artwork_url')
451 thumbnail = artwork_url or user.get('avatar_url')
452 if isinstance(thumbnail, compat_str):
453 if re.search(self._IMAGE_REPL_RE, thumbnail):
454 for image_id, size in self._ARTWORK_MAP.items():
455 i = {
456 'id': image_id,
457 'url': re.sub(self._IMAGE_REPL_RE, '-%s.jpg' % image_id, thumbnail),
458 }
459 if image_id == 'tiny' and not artwork_url:
460 size = 18
461 elif image_id == 'original':
462 i['preference'] = 10
463 if size:
464 i.update({
465 'width': size,
466 'height': size,
467 })
468 thumbnails.append(i)
469 else:
470 thumbnails = [{'url': thumbnail}]
471
472 def extract_count(key):
473 return int_or_none(info.get('%s_count' % key))
474
475 return {
476 'id': track_id,
477 'uploader': user.get('username'),
478 'uploader_id': str_or_none(user.get('id')) or user.get('permalink'),
479 'uploader_url': user.get('permalink_url'),
480 'timestamp': unified_timestamp(info.get('created_at')),
481 'title': title,
482 'description': info.get('description'),
483 'thumbnails': thumbnails,
484 'duration': float_or_none(info.get('duration'), 1000),
485 'webpage_url': info.get('permalink_url'),
486 'license': info.get('license'),
487 'view_count': extract_count('playback'),
488 'like_count': extract_count('favoritings') or extract_count('likes'),
489 'comment_count': extract_count('comment'),
490 'repost_count': extract_count('reposts'),
491 'genre': info.get('genre'),
492 'formats': formats
493 }
494
495 def _real_extract(self, url):
496 mobj = re.match(self._VALID_URL, url)
497
498 track_id = mobj.group('track_id')
499
500 query = {}
501 if track_id:
502 info_json_url = self._API_V2_BASE + 'tracks/' + track_id
503 full_title = track_id
504 token = mobj.group('secret_token')
505 if token:
506 query['secret_token'] = token
507 else:
508 full_title = resolve_title = '%s/%s' % mobj.group('uploader', 'title')
509 token = mobj.group('token')
510 if token:
511 resolve_title += '/%s' % token
512 info_json_url = self._resolv_url(self._BASE_URL + resolve_title)
513
514 version = 2
515 info = self._download_json(
516 info_json_url, full_title, 'Downloading info JSON', query=query, fatal=False)
517 if not info:
518 info = self._download_json(
519 info_json_url.replace(self._API_V2_BASE, self._API_BASE),
520 full_title, 'Downloading info JSON', query=query)
521 version = 1
522
523 return self._extract_info_dict(info, full_title, token, version)
524
525
526 class SoundcloudPlaylistBaseIE(SoundcloudIE):
527 def _extract_track_entries(self, tracks, token=None):
528 entries = []
529 for track in tracks:
530 track_id = str_or_none(track.get('id'))
531 url = track.get('permalink_url')
532 if not url:
533 if not track_id:
534 continue
535 url = self._API_V2_BASE + 'tracks/' + track_id
536 if token:
537 url += '?secret_token=' + token
538 entries.append(self.url_result(
539 url, SoundcloudIE.ie_key(), track_id))
540 return entries
541
542
543 class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
544 _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
545 IE_NAME = 'soundcloud:set'
546 _TESTS = [{
547 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
548 'info_dict': {
549 'id': '2284613',
550 'title': 'The Royal Concept EP',
551 },
552 'playlist_mincount': 5,
553 }, {
554 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
555 'only_matching': True,
556 }]
557
558 def _real_extract(self, url):
559 mobj = re.match(self._VALID_URL, url)
560
561 full_title = '%s/sets/%s' % mobj.group('uploader', 'slug_title')
562 token = mobj.group('token')
563 if token:
564 full_title += '/' + token
565
566 info = self._download_json(self._resolv_url(
567 self._BASE_URL + full_title), full_title)
568
569 if 'errors' in info:
570 msgs = (compat_str(err['error_message']) for err in info['errors'])
571 raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
572
573 entries = self._extract_track_entries(info['tracks'], token)
574
575 return self.playlist_result(
576 entries, str_or_none(info.get('id')), info.get('title'))
577
578
579 class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
580 def _extract_playlist(self, base_url, playlist_id, playlist_title):
581 COMMON_QUERY = {
582 'limit': 2000000000,
583 'linked_partitioning': '1',
584 }
585
586 query = COMMON_QUERY.copy()
587 query['offset'] = 0
588
589 next_href = base_url
590
591 entries = []
592 for i in itertools.count():
593 response = self._download_json(
594 next_href, playlist_id,
595 'Downloading track page %s' % (i + 1), query=query)
596
597 collection = response['collection']
598
599 if not isinstance(collection, list):
600 collection = []
601
602 # Empty collection may be returned, in this case we proceed
603 # straight to next_href
604
605 def resolve_entry(candidates):
606 for cand in candidates:
607 if not isinstance(cand, dict):
608 continue
609 permalink_url = url_or_none(cand.get('permalink_url'))
610 if not permalink_url:
611 continue
612 return self.url_result(
613 permalink_url,
614 SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
615 str_or_none(cand.get('id')), cand.get('title'))
616
617 for e in collection:
618 entry = resolve_entry((e, e.get('track'), e.get('playlist')))
619 if entry:
620 entries.append(entry)
621
622 next_href = response.get('next_href')
623 if not next_href:
624 break
625
626 next_href = response['next_href']
627 parsed_next_href = compat_urlparse.urlparse(next_href)
628 query = compat_urlparse.parse_qs(parsed_next_href.query)
629 query.update(COMMON_QUERY)
630
631 return {
632 '_type': 'playlist',
633 'id': playlist_id,
634 'title': playlist_title,
635 'entries': entries,
636 }
637
638
639 class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
640 _VALID_URL = r'''(?x)
641 https?://
642 (?:(?:www|m)\.)?soundcloud\.com/
643 (?P<user>[^/]+)
644 (?:/
645 (?P<rsrc>tracks|albums|sets|reposts|likes|spotlight)
646 )?
647 /?(?:[?#].*)?$
648 '''
649 IE_NAME = 'soundcloud:user'
650 _TESTS = [{
651 'url': 'https://soundcloud.com/soft-cell-official',
652 'info_dict': {
653 'id': '207965082',
654 'title': 'Soft Cell (All)',
655 },
656 'playlist_mincount': 28,
657 }, {
658 'url': 'https://soundcloud.com/soft-cell-official/tracks',
659 'info_dict': {
660 'id': '207965082',
661 'title': 'Soft Cell (Tracks)',
662 },
663 'playlist_mincount': 27,
664 }, {
665 'url': 'https://soundcloud.com/soft-cell-official/albums',
666 'info_dict': {
667 'id': '207965082',
668 'title': 'Soft Cell (Albums)',
669 },
670 'playlist_mincount': 1,
671 }, {
672 'url': 'https://soundcloud.com/jcv246/sets',
673 'info_dict': {
674 'id': '12982173',
675 'title': 'Jordi / cv (Sets)',
676 },
677 'playlist_mincount': 2,
678 }, {
679 'url': 'https://soundcloud.com/jcv246/reposts',
680 'info_dict': {
681 'id': '12982173',
682 'title': 'Jordi / cv (Reposts)',
683 },
684 'playlist_mincount': 6,
685 }, {
686 'url': 'https://soundcloud.com/clalberg/likes',
687 'info_dict': {
688 'id': '11817582',
689 'title': 'clalberg (Likes)',
690 },
691 'playlist_mincount': 5,
692 }, {
693 'url': 'https://soundcloud.com/grynpyret/spotlight',
694 'info_dict': {
695 'id': '7098329',
696 'title': 'Grynpyret (Spotlight)',
697 },
698 'playlist_mincount': 1,
699 }]
700
701 _BASE_URL_MAP = {
702 'all': 'stream/users/%s',
703 'tracks': 'users/%s/tracks',
704 'albums': 'users/%s/albums',
705 'sets': 'users/%s/playlists',
706 'reposts': 'stream/users/%s/reposts',
707 'likes': 'users/%s/likes',
708 'spotlight': 'users/%s/spotlight',
709 }
710
711 def _real_extract(self, url):
712 mobj = re.match(self._VALID_URL, url)
713 uploader = mobj.group('user')
714
715 user = self._download_json(
716 self._resolv_url(self._BASE_URL + uploader),
717 uploader, 'Downloading user info')
718
719 resource = mobj.group('rsrc') or 'all'
720
721 return self._extract_playlist(
722 self._API_V2_BASE + self._BASE_URL_MAP[resource] % user['id'],
723 str_or_none(user.get('id')),
724 '%s (%s)' % (user['username'], resource.capitalize()))
725
726
727 class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
728 _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
729 IE_NAME = 'soundcloud:trackstation'
730 _TESTS = [{
731 'url': 'https://soundcloud.com/stations/track/officialsundial/your-text',
732 'info_dict': {
733 'id': '286017854',
734 'title': 'Track station: your text',
735 },
736 'playlist_mincount': 47,
737 }]
738
739 def _real_extract(self, url):
740 track_name = self._match_id(url)
741
742 track = self._download_json(self._resolv_url(url), track_name)
743 track_id = self._search_regex(
744 r'soundcloud:track-stations:(\d+)', track['id'], 'track id')
745
746 return self._extract_playlist(
747 self._API_V2_BASE + 'stations/%s/tracks' % track['id'],
748 track_id, 'Track station: %s' % track['title'])
749
750
751 class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
752 _VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
753 IE_NAME = 'soundcloud:playlist'
754 _TESTS = [{
755 'url': 'https://api.soundcloud.com/playlists/4110309',
756 'info_dict': {
757 'id': '4110309',
758 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
759 'description': 're:.*?TILT Brass - Bowery Poetry Club',
760 },
761 'playlist_count': 6,
762 }]
763
764 def _real_extract(self, url):
765 mobj = re.match(self._VALID_URL, url)
766 playlist_id = mobj.group('id')
767
768 query = {}
769 token = mobj.group('token')
770 if token:
771 query['secret_token'] = token
772
773 data = self._download_json(
774 self._API_V2_BASE + 'playlists/' + playlist_id,
775 playlist_id, 'Downloading playlist', query=query)
776
777 entries = self._extract_track_entries(data['tracks'], token)
778
779 return self.playlist_result(
780 entries, playlist_id, data.get('title'), data.get('description'))
781
782
783 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
784 IE_NAME = 'soundcloud:search'
785 IE_DESC = 'Soundcloud search'
786 _MAX_RESULTS = float('inf')
787 _TESTS = [{
788 'url': 'scsearch15:post-avant jazzcore',
789 'info_dict': {
790 'title': 'post-avant jazzcore',
791 },
792 'playlist_count': 15,
793 }]
794
795 _SEARCH_KEY = 'scsearch'
796 _MAX_RESULTS_PER_PAGE = 200
797 _DEFAULT_RESULTS_PER_PAGE = 50
798
799 def _get_collection(self, endpoint, collection_id, **query):
800 limit = min(
801 query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
802 self._MAX_RESULTS_PER_PAGE)
803 query.update({
804 'limit': limit,
805 'linked_partitioning': 1,
806 'offset': 0,
807 })
808 next_url = update_url_query(self._API_V2_BASE + endpoint, query)
809
810 collected_results = 0
811
812 for i in itertools.count(1):
813 response = self._download_json(
814 next_url, collection_id, 'Downloading page {0}'.format(i),
815 'Unable to download API page')
816
817 collection = response.get('collection', [])
818 if not collection:
819 break
820
821 collection = list(filter(bool, collection))
822 collected_results += len(collection)
823
824 for item in collection:
825 yield self.url_result(item['uri'], SoundcloudIE.ie_key())
826
827 if not collection or collected_results >= limit:
828 break
829
830 next_url = response.get('next_href')
831 if not next_url:
832 break
833
834 def _get_n_results(self, query, n):
835 tracks = self._get_collection('search/tracks', query, limit=n, q=query)
836 return self.playlist_result(tracks, playlist_title=query)