]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/soundcloud.py
Set upstream metadata fields: Bug-Database, Bug-Submit, Repository, Repository-Browse.
[youtubedl] / youtube_dl / extractor / soundcloud.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import itertools
5 import re
6
7 from .common import (
8 InfoExtractor,
9 SearchInfoExtractor
10 )
11 from ..compat import (
12 compat_HTTPError,
13 compat_kwargs,
14 compat_str,
15 compat_urlparse,
16 )
17 from ..utils import (
18 error_to_compat_str,
19 ExtractorError,
20 float_or_none,
21 HEADRequest,
22 int_or_none,
23 KNOWN_EXTENSIONS,
24 mimetype2ext,
25 str_or_none,
26 try_get,
27 unified_timestamp,
28 update_url_query,
29 url_or_none,
30 urlhandle_detect_ext,
31 )
32
33
34 class SoundcloudEmbedIE(InfoExtractor):
35 _VALID_URL = r'https?://(?:w|player|p)\.soundcloud\.com/player/?.*?\burl=(?P<id>.+)'
36 _TEST = {
37 # from https://www.soundi.fi/uutiset/ennakkokuuntelussa-timo-kaukolammen-station-to-station-to-station-julkaisua-juhlitaan-tanaan-g-livelabissa/
38 'url': 'https://w.soundcloud.com/player/?visual=true&url=https%3A%2F%2Fapi.soundcloud.com%2Fplaylists%2F922213810&show_artwork=true&maxwidth=640&maxheight=960&dnt=1&secret_token=s-ziYey',
39 'only_matching': True,
40 }
41
42 @staticmethod
43 def _extract_urls(webpage):
44 return [m.group('url') for m in re.finditer(
45 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:w\.)?soundcloud\.com/player.+?)\1',
46 webpage)]
47
48 def _real_extract(self, url):
49 query = compat_urlparse.parse_qs(
50 compat_urlparse.urlparse(url).query)
51 api_url = query['url'][0]
52 secret_token = query.get('secret_token')
53 if secret_token:
54 api_url = update_url_query(api_url, {'secret_token': secret_token[0]})
55 return self.url_result(api_url)
56
57
58 class SoundcloudIE(InfoExtractor):
59 """Information extractor for soundcloud.com
60 To access the media, the uid of the song and a stream token
61 must be extracted from the page source and the script must make
62 a request to media.soundcloud.com/crossdomain.xml. Then
63 the media can be grabbed by requesting from an url composed
64 of the stream token and uid
65 """
66
67 _VALID_URL = r'''(?x)^(?:https?://)?
68 (?:(?:(?:www\.|m\.)?soundcloud\.com/
69 (?!stations/track)
70 (?P<uploader>[\w\d-]+)/
71 (?!(?:tracks|albums|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
72 (?P<title>[\w\d-]+)/?
73 (?P<token>[^?]+?)?(?:[?].*)?$)
74 |(?:api(?:-v2)?\.soundcloud\.com/tracks/(?P<track_id>\d+)
75 (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
76 )
77 '''
78 IE_NAME = 'soundcloud'
79 _TESTS = [
80 {
81 'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
82 'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
83 'info_dict': {
84 'id': '62986583',
85 'ext': 'mp3',
86 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
87 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
88 'uploader': 'E.T. ExTerrestrial Music',
89 'uploader_id': '1571244',
90 'timestamp': 1349920598,
91 'upload_date': '20121011',
92 'duration': 143.216,
93 'license': 'all-rights-reserved',
94 'view_count': int,
95 'like_count': int,
96 'comment_count': int,
97 'repost_count': int,
98 }
99 },
100 # geo-restricted
101 {
102 'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
103 'info_dict': {
104 'id': '47127627',
105 'ext': 'mp3',
106 'title': 'Goldrushed',
107 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
108 'uploader': 'The Royal Concept',
109 'uploader_id': '9615865',
110 'timestamp': 1337635207,
111 'upload_date': '20120521',
112 'duration': 227.155,
113 'license': 'all-rights-reserved',
114 'view_count': int,
115 'like_count': int,
116 'comment_count': int,
117 'repost_count': int,
118 },
119 },
120 # private link
121 {
122 'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
123 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
124 'info_dict': {
125 'id': '123998367',
126 'ext': 'mp3',
127 'title': 'Youtube - Dl Test Video \'\' Ä↭',
128 'description': 'test chars: \"\'/\\ä↭',
129 'uploader': 'jaimeMF',
130 'uploader_id': '69767071',
131 'timestamp': 1386604920,
132 'upload_date': '20131209',
133 'duration': 9.927,
134 'license': 'all-rights-reserved',
135 'view_count': int,
136 'like_count': int,
137 'comment_count': int,
138 'repost_count': int,
139 },
140 },
141 # private link (alt format)
142 {
143 'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp',
144 'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
145 'info_dict': {
146 'id': '123998367',
147 'ext': 'mp3',
148 'title': 'Youtube - Dl Test Video \'\' Ä↭',
149 'description': 'test chars: \"\'/\\ä↭',
150 'uploader': 'jaimeMF',
151 'uploader_id': '69767071',
152 'timestamp': 1386604920,
153 'upload_date': '20131209',
154 'duration': 9.927,
155 'license': 'all-rights-reserved',
156 'view_count': int,
157 'like_count': int,
158 'comment_count': int,
159 'repost_count': int,
160 },
161 },
162 # downloadable song
163 {
164 'url': 'https://soundcloud.com/oddsamples/bus-brakes',
165 'md5': '7624f2351f8a3b2e7cd51522496e7631',
166 'info_dict': {
167 'id': '128590877',
168 'ext': 'mp3',
169 'title': 'Bus Brakes',
170 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
171 'uploader': 'oddsamples',
172 'uploader_id': '73680509',
173 'timestamp': 1389232924,
174 'upload_date': '20140109',
175 'duration': 17.346,
176 'license': 'cc-by-sa',
177 'view_count': int,
178 'like_count': int,
179 'comment_count': int,
180 'repost_count': int,
181 },
182 },
183 # private link, downloadable format
184 {
185 'url': 'https://soundcloud.com/oriuplift/uponly-238-no-talking-wav/s-AyZUd',
186 'md5': '64a60b16e617d41d0bef032b7f55441e',
187 'info_dict': {
188 'id': '340344461',
189 'ext': 'wav',
190 'title': 'Uplifting Only 238 [No Talking] (incl. Alex Feed Guestmix) (Aug 31, 2017) [wav]',
191 'description': 'md5:fa20ee0fca76a3d6df8c7e57f3715366',
192 'uploader': 'Ori Uplift Music',
193 'uploader_id': '12563093',
194 'timestamp': 1504206263,
195 'upload_date': '20170831',
196 'duration': 7449.096,
197 'license': 'all-rights-reserved',
198 'view_count': int,
199 'like_count': int,
200 'comment_count': int,
201 'repost_count': int,
202 },
203 },
204 # no album art, use avatar pic for thumbnail
205 {
206 'url': 'https://soundcloud.com/garyvee/sideways-prod-mad-real',
207 'md5': '59c7872bc44e5d99b7211891664760c2',
208 'info_dict': {
209 'id': '309699954',
210 'ext': 'mp3',
211 'title': 'Sideways (Prod. Mad Real)',
212 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
213 'uploader': 'garyvee',
214 'uploader_id': '2366352',
215 'timestamp': 1488152409,
216 'upload_date': '20170226',
217 'duration': 207.012,
218 'thumbnail': r're:https?://.*\.jpg',
219 'license': 'all-rights-reserved',
220 'view_count': int,
221 'like_count': int,
222 'comment_count': int,
223 'repost_count': int,
224 },
225 'params': {
226 'skip_download': True,
227 },
228 },
229 {
230 'url': 'https://soundcloud.com/giovannisarani/mezzo-valzer',
231 'md5': 'e22aecd2bc88e0e4e432d7dcc0a1abf7',
232 'info_dict': {
233 'id': '583011102',
234 'ext': 'mp3',
235 'title': 'Mezzo Valzer',
236 'description': 'md5:4138d582f81866a530317bae316e8b61',
237 'uploader': 'Micronie',
238 'uploader_id': '3352531',
239 'timestamp': 1551394171,
240 'upload_date': '20190228',
241 'duration': 180.157,
242 'thumbnail': r're:https?://.*\.jpg',
243 'license': 'all-rights-reserved',
244 'view_count': int,
245 'like_count': int,
246 'comment_count': int,
247 'repost_count': int,
248 },
249 }
250 ]
251
252 _API_V2_BASE = 'https://api-v2.soundcloud.com/'
253 _BASE_URL = 'https://soundcloud.com/'
254 _IMAGE_REPL_RE = r'-([0-9a-z]+)\.jpg'
255
256 _ARTWORK_MAP = {
257 'mini': 16,
258 'tiny': 20,
259 'small': 32,
260 'badge': 47,
261 't67x67': 67,
262 'large': 100,
263 't300x300': 300,
264 'crop': 400,
265 't500x500': 500,
266 'original': 0,
267 }
268
269 def _store_client_id(self, client_id):
270 self._downloader.cache.store('soundcloud', 'client_id', client_id)
271
272 def _update_client_id(self):
273 webpage = self._download_webpage('https://soundcloud.com/', None)
274 for src in reversed(re.findall(r'<script[^>]+src="([^"]+)"', webpage)):
275 script = self._download_webpage(src, None, fatal=False)
276 if script:
277 client_id = self._search_regex(
278 r'client_id\s*:\s*"([0-9a-zA-Z]{32})"',
279 script, 'client id', default=None)
280 if client_id:
281 self._CLIENT_ID = client_id
282 self._store_client_id(client_id)
283 return
284 raise ExtractorError('Unable to extract client id')
285
286 def _download_json(self, *args, **kwargs):
287 non_fatal = kwargs.get('fatal') is False
288 if non_fatal:
289 del kwargs['fatal']
290 query = kwargs.get('query', {}).copy()
291 for _ in range(2):
292 query['client_id'] = self._CLIENT_ID
293 kwargs['query'] = query
294 try:
295 return super(SoundcloudIE, self)._download_json(*args, **compat_kwargs(kwargs))
296 except ExtractorError as e:
297 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
298 self._store_client_id(None)
299 self._update_client_id()
300 continue
301 elif non_fatal:
302 self._downloader.report_warning(error_to_compat_str(e))
303 return False
304 raise
305
306 def _real_initialize(self):
307 self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk'
308
309 @classmethod
310 def _resolv_url(cls, url):
311 return SoundcloudIE._API_V2_BASE + 'resolve?url=' + url
312
313 def _extract_info_dict(self, info, full_title=None, secret_token=None):
314 track_id = compat_str(info['id'])
315 title = info['title']
316
317 format_urls = set()
318 formats = []
319 query = {'client_id': self._CLIENT_ID}
320 if secret_token:
321 query['secret_token'] = secret_token
322
323 if info.get('downloadable') and info.get('has_downloads_left'):
324 download_url = update_url_query(
325 self._API_V2_BASE + 'tracks/' + track_id + '/download', query)
326 redirect_url = (self._download_json(download_url, track_id, fatal=False) or {}).get('redirectUri')
327 if redirect_url:
328 urlh = self._request_webpage(
329 HEADRequest(redirect_url), track_id, fatal=False)
330 if urlh:
331 format_url = urlh.geturl()
332 format_urls.add(format_url)
333 formats.append({
334 'format_id': 'download',
335 'ext': urlhandle_detect_ext(urlh) or 'mp3',
336 'filesize': int_or_none(urlh.headers.get('Content-Length')),
337 'url': format_url,
338 'preference': 10,
339 })
340
341 def invalid_url(url):
342 return not url or url in format_urls
343
344 def add_format(f, protocol, is_preview=False):
345 mobj = re.search(r'\.(?P<abr>\d+)\.(?P<ext>[0-9a-z]{3,4})(?=[/?])', stream_url)
346 if mobj:
347 for k, v in mobj.groupdict().items():
348 if not f.get(k):
349 f[k] = v
350 format_id_list = []
351 if protocol:
352 format_id_list.append(protocol)
353 for k in ('ext', 'abr'):
354 v = f.get(k)
355 if v:
356 format_id_list.append(v)
357 preview = is_preview or re.search(r'/(?:preview|playlist)/0/30/', f['url'])
358 if preview:
359 format_id_list.append('preview')
360 abr = f.get('abr')
361 if abr:
362 f['abr'] = int(abr)
363 f.update({
364 'format_id': '_'.join(format_id_list),
365 'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
366 'preference': -10 if preview else None,
367 })
368 formats.append(f)
369
370 # New API
371 transcodings = try_get(
372 info, lambda x: x['media']['transcodings'], list) or []
373 for t in transcodings:
374 if not isinstance(t, dict):
375 continue
376 format_url = url_or_none(t.get('url'))
377 if not format_url:
378 continue
379 stream = self._download_json(
380 format_url, track_id, query=query, fatal=False)
381 if not isinstance(stream, dict):
382 continue
383 stream_url = url_or_none(stream.get('url'))
384 if invalid_url(stream_url):
385 continue
386 format_urls.add(stream_url)
387 stream_format = t.get('format') or {}
388 protocol = stream_format.get('protocol')
389 if protocol != 'hls' and '/hls' in format_url:
390 protocol = 'hls'
391 ext = None
392 preset = str_or_none(t.get('preset'))
393 if preset:
394 ext = preset.split('_')[0]
395 if ext not in KNOWN_EXTENSIONS:
396 ext = mimetype2ext(stream_format.get('mime_type'))
397 add_format({
398 'url': stream_url,
399 'ext': ext,
400 }, 'http' if protocol == 'progressive' else protocol,
401 t.get('snipped') or '/preview/' in format_url)
402
403 for f in formats:
404 f['vcodec'] = 'none'
405
406 if not formats and info.get('policy') == 'BLOCK':
407 self.raise_geo_restricted()
408 self._sort_formats(formats)
409
410 user = info.get('user') or {}
411
412 thumbnails = []
413 artwork_url = info.get('artwork_url')
414 thumbnail = artwork_url or user.get('avatar_url')
415 if isinstance(thumbnail, compat_str):
416 if re.search(self._IMAGE_REPL_RE, thumbnail):
417 for image_id, size in self._ARTWORK_MAP.items():
418 i = {
419 'id': image_id,
420 'url': re.sub(self._IMAGE_REPL_RE, '-%s.jpg' % image_id, thumbnail),
421 }
422 if image_id == 'tiny' and not artwork_url:
423 size = 18
424 elif image_id == 'original':
425 i['preference'] = 10
426 if size:
427 i.update({
428 'width': size,
429 'height': size,
430 })
431 thumbnails.append(i)
432 else:
433 thumbnails = [{'url': thumbnail}]
434
435 def extract_count(key):
436 return int_or_none(info.get('%s_count' % key))
437
438 return {
439 'id': track_id,
440 'uploader': user.get('username'),
441 'uploader_id': str_or_none(user.get('id')) or user.get('permalink'),
442 'uploader_url': user.get('permalink_url'),
443 'timestamp': unified_timestamp(info.get('created_at')),
444 'title': title,
445 'description': info.get('description'),
446 'thumbnails': thumbnails,
447 'duration': float_or_none(info.get('duration'), 1000),
448 'webpage_url': info.get('permalink_url'),
449 'license': info.get('license'),
450 'view_count': extract_count('playback'),
451 'like_count': extract_count('favoritings') or extract_count('likes'),
452 'comment_count': extract_count('comment'),
453 'repost_count': extract_count('reposts'),
454 'genre': info.get('genre'),
455 'formats': formats
456 }
457
458 def _real_extract(self, url):
459 mobj = re.match(self._VALID_URL, url)
460
461 track_id = mobj.group('track_id')
462
463 query = {}
464 if track_id:
465 info_json_url = self._API_V2_BASE + 'tracks/' + track_id
466 full_title = track_id
467 token = mobj.group('secret_token')
468 if token:
469 query['secret_token'] = token
470 else:
471 full_title = resolve_title = '%s/%s' % mobj.group('uploader', 'title')
472 token = mobj.group('token')
473 if token:
474 resolve_title += '/%s' % token
475 info_json_url = self._resolv_url(self._BASE_URL + resolve_title)
476
477 info = self._download_json(
478 info_json_url, full_title, 'Downloading info JSON', query=query)
479
480 return self._extract_info_dict(info, full_title, token)
481
482
483 class SoundcloudPlaylistBaseIE(SoundcloudIE):
484 def _extract_set(self, playlist, token=None):
485 playlist_id = compat_str(playlist['id'])
486 tracks = playlist.get('tracks') or []
487 if not all([t.get('permalink_url') for t in tracks]) and token:
488 tracks = self._download_json(
489 self._API_V2_BASE + 'tracks', playlist_id,
490 'Downloading tracks', query={
491 'ids': ','.join([compat_str(t['id']) for t in tracks]),
492 'playlistId': playlist_id,
493 'playlistSecretToken': token,
494 })
495 entries = []
496 for track in tracks:
497 track_id = str_or_none(track.get('id'))
498 url = track.get('permalink_url')
499 if not url:
500 if not track_id:
501 continue
502 url = self._API_V2_BASE + 'tracks/' + track_id
503 if token:
504 url += '?secret_token=' + token
505 entries.append(self.url_result(
506 url, SoundcloudIE.ie_key(), track_id))
507 return self.playlist_result(
508 entries, playlist_id,
509 playlist.get('title'),
510 playlist.get('description'))
511
512
513 class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
514 _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
515 IE_NAME = 'soundcloud:set'
516 _TESTS = [{
517 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
518 'info_dict': {
519 'id': '2284613',
520 'title': 'The Royal Concept EP',
521 'description': 'md5:71d07087c7a449e8941a70a29e34671e',
522 },
523 'playlist_mincount': 5,
524 }, {
525 'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
526 'only_matching': True,
527 }]
528
529 def _real_extract(self, url):
530 mobj = re.match(self._VALID_URL, url)
531
532 full_title = '%s/sets/%s' % mobj.group('uploader', 'slug_title')
533 token = mobj.group('token')
534 if token:
535 full_title += '/' + token
536
537 info = self._download_json(self._resolv_url(
538 self._BASE_URL + full_title), full_title)
539
540 if 'errors' in info:
541 msgs = (compat_str(err['error_message']) for err in info['errors'])
542 raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
543
544 return self._extract_set(info, token)
545
546
547 class SoundcloudPagedPlaylistBaseIE(SoundcloudIE):
548 def _extract_playlist(self, base_url, playlist_id, playlist_title):
549 COMMON_QUERY = {
550 'limit': 2000000000,
551 'linked_partitioning': '1',
552 }
553
554 query = COMMON_QUERY.copy()
555 query['offset'] = 0
556
557 next_href = base_url
558
559 entries = []
560 for i in itertools.count():
561 response = self._download_json(
562 next_href, playlist_id,
563 'Downloading track page %s' % (i + 1), query=query)
564
565 collection = response['collection']
566
567 if not isinstance(collection, list):
568 collection = []
569
570 # Empty collection may be returned, in this case we proceed
571 # straight to next_href
572
573 def resolve_entry(candidates):
574 for cand in candidates:
575 if not isinstance(cand, dict):
576 continue
577 permalink_url = url_or_none(cand.get('permalink_url'))
578 if not permalink_url:
579 continue
580 return self.url_result(
581 permalink_url,
582 SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
583 str_or_none(cand.get('id')), cand.get('title'))
584
585 for e in collection:
586 entry = resolve_entry((e, e.get('track'), e.get('playlist')))
587 if entry:
588 entries.append(entry)
589
590 next_href = response.get('next_href')
591 if not next_href:
592 break
593
594 next_href = response['next_href']
595 parsed_next_href = compat_urlparse.urlparse(next_href)
596 query = compat_urlparse.parse_qs(parsed_next_href.query)
597 query.update(COMMON_QUERY)
598
599 return {
600 '_type': 'playlist',
601 'id': playlist_id,
602 'title': playlist_title,
603 'entries': entries,
604 }
605
606
607 class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
608 _VALID_URL = r'''(?x)
609 https?://
610 (?:(?:www|m)\.)?soundcloud\.com/
611 (?P<user>[^/]+)
612 (?:/
613 (?P<rsrc>tracks|albums|sets|reposts|likes|spotlight)
614 )?
615 /?(?:[?#].*)?$
616 '''
617 IE_NAME = 'soundcloud:user'
618 _TESTS = [{
619 'url': 'https://soundcloud.com/soft-cell-official',
620 'info_dict': {
621 'id': '207965082',
622 'title': 'Soft Cell (All)',
623 },
624 'playlist_mincount': 28,
625 }, {
626 'url': 'https://soundcloud.com/soft-cell-official/tracks',
627 'info_dict': {
628 'id': '207965082',
629 'title': 'Soft Cell (Tracks)',
630 },
631 'playlist_mincount': 27,
632 }, {
633 'url': 'https://soundcloud.com/soft-cell-official/albums',
634 'info_dict': {
635 'id': '207965082',
636 'title': 'Soft Cell (Albums)',
637 },
638 'playlist_mincount': 1,
639 }, {
640 'url': 'https://soundcloud.com/jcv246/sets',
641 'info_dict': {
642 'id': '12982173',
643 'title': 'Jordi / cv (Sets)',
644 },
645 'playlist_mincount': 2,
646 }, {
647 'url': 'https://soundcloud.com/jcv246/reposts',
648 'info_dict': {
649 'id': '12982173',
650 'title': 'Jordi / cv (Reposts)',
651 },
652 'playlist_mincount': 6,
653 }, {
654 'url': 'https://soundcloud.com/clalberg/likes',
655 'info_dict': {
656 'id': '11817582',
657 'title': 'clalberg (Likes)',
658 },
659 'playlist_mincount': 5,
660 }, {
661 'url': 'https://soundcloud.com/grynpyret/spotlight',
662 'info_dict': {
663 'id': '7098329',
664 'title': 'Grynpyret (Spotlight)',
665 },
666 'playlist_mincount': 1,
667 }]
668
669 _BASE_URL_MAP = {
670 'all': 'stream/users/%s',
671 'tracks': 'users/%s/tracks',
672 'albums': 'users/%s/albums',
673 'sets': 'users/%s/playlists',
674 'reposts': 'stream/users/%s/reposts',
675 'likes': 'users/%s/likes',
676 'spotlight': 'users/%s/spotlight',
677 }
678
679 def _real_extract(self, url):
680 mobj = re.match(self._VALID_URL, url)
681 uploader = mobj.group('user')
682
683 user = self._download_json(
684 self._resolv_url(self._BASE_URL + uploader),
685 uploader, 'Downloading user info')
686
687 resource = mobj.group('rsrc') or 'all'
688
689 return self._extract_playlist(
690 self._API_V2_BASE + self._BASE_URL_MAP[resource] % user['id'],
691 str_or_none(user.get('id')),
692 '%s (%s)' % (user['username'], resource.capitalize()))
693
694
695 class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
696 _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
697 IE_NAME = 'soundcloud:trackstation'
698 _TESTS = [{
699 'url': 'https://soundcloud.com/stations/track/officialsundial/your-text',
700 'info_dict': {
701 'id': '286017854',
702 'title': 'Track station: your text',
703 },
704 'playlist_mincount': 47,
705 }]
706
707 def _real_extract(self, url):
708 track_name = self._match_id(url)
709
710 track = self._download_json(self._resolv_url(url), track_name)
711 track_id = self._search_regex(
712 r'soundcloud:track-stations:(\d+)', track['id'], 'track id')
713
714 return self._extract_playlist(
715 self._API_V2_BASE + 'stations/%s/tracks' % track['id'],
716 track_id, 'Track station: %s' % track['title'])
717
718
719 class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
720 _VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
721 IE_NAME = 'soundcloud:playlist'
722 _TESTS = [{
723 'url': 'https://api.soundcloud.com/playlists/4110309',
724 'info_dict': {
725 'id': '4110309',
726 'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
727 'description': 're:.*?TILT Brass - Bowery Poetry Club',
728 },
729 'playlist_count': 6,
730 }]
731
732 def _real_extract(self, url):
733 mobj = re.match(self._VALID_URL, url)
734 playlist_id = mobj.group('id')
735
736 query = {}
737 token = mobj.group('token')
738 if token:
739 query['secret_token'] = token
740
741 data = self._download_json(
742 self._API_V2_BASE + 'playlists/' + playlist_id,
743 playlist_id, 'Downloading playlist', query=query)
744
745 return self._extract_set(data, token)
746
747
748 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
749 IE_NAME = 'soundcloud:search'
750 IE_DESC = 'Soundcloud search'
751 _MAX_RESULTS = float('inf')
752 _TESTS = [{
753 'url': 'scsearch15:post-avant jazzcore',
754 'info_dict': {
755 'title': 'post-avant jazzcore',
756 },
757 'playlist_count': 15,
758 }]
759
760 _SEARCH_KEY = 'scsearch'
761 _MAX_RESULTS_PER_PAGE = 200
762 _DEFAULT_RESULTS_PER_PAGE = 50
763
764 def _get_collection(self, endpoint, collection_id, **query):
765 limit = min(
766 query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
767 self._MAX_RESULTS_PER_PAGE)
768 query.update({
769 'limit': limit,
770 'linked_partitioning': 1,
771 'offset': 0,
772 })
773 next_url = update_url_query(self._API_V2_BASE + endpoint, query)
774
775 collected_results = 0
776
777 for i in itertools.count(1):
778 response = self._download_json(
779 next_url, collection_id, 'Downloading page {0}'.format(i),
780 'Unable to download API page')
781
782 collection = response.get('collection', [])
783 if not collection:
784 break
785
786 collection = list(filter(bool, collection))
787 collected_results += len(collection)
788
789 for item in collection:
790 yield self.url_result(item['uri'], SoundcloudIE.ie_key())
791
792 if not collection or collected_results >= limit:
793 break
794
795 next_url = response.get('next_href')
796 if not next_url:
797 break
798
799 def _get_n_results(self, query, n):
800 tracks = self._get_collection('search/tracks', query, limit=n, q=query)
801 return self.playlist_result(tracks, playlist_title=query)