2 from __future__ 
import unicode_literals
 
  11 from ..compat 
import ( 
  14     compat_urllib_parse_urlencode
, 
  24 class SoundcloudIE(InfoExtractor
): 
  25     """Information extractor for soundcloud.com 
  26        To access the media, the uid of the song and a stream token 
  27        must be extracted from the page source and the script must make 
  28        a request to media.soundcloud.com/crossdomain.xml. Then 
  29        the media can be grabbed by requesting from an url composed 
  30        of the stream token and uid 
  33     _VALID_URL 
= r
'''(?x)^(?:https?://)? 
  34                     (?:(?:(?:www\.|m\.)?soundcloud\.com/ 
  36                             (?P<uploader>[\w\d-]+)/ 
  37                             (?!(?:tracks|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#])) 
  39                             (?P<token>[^?]+?)?(?:[?].*)?$) 
  40                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+) 
  41                           (?:/?\?secret_token=(?P<secret_token>[^&]+))?) 
  42                        |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*) 
  45     IE_NAME 
= 'soundcloud' 
  48             'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy', 
  49             'md5': 'ebef0a451b909710ed1d7787dddbf0d7', 
  53                 'upload_date': '20121011', 
  54                 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d', 
  55                 'uploader': 'E.T. ExTerrestrial Music', 
  56                 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1', 
  58                 'license': 'all-rights-reserved', 
  63             'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep', 
  67                 'title': 'Goldrushed', 
  68                 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com', 
  69                 'uploader': 'The Royal Concept', 
  70                 'upload_date': '20120521', 
  72                 'license': 'all-rights-reserved', 
  76                 'skip_download': True, 
  81             'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp', 
  82             'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604', 
  86                 'title': 'Youtube - Dl Test Video \'\' Ä↭', 
  87                 'uploader': 'jaimeMF', 
  88                 'description': 'test chars:  \"\'/\\ä↭', 
  89                 'upload_date': '20131209', 
  91                 'license': 'all-rights-reserved', 
  94         # private link (alt format) 
  96             'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp', 
  97             'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604', 
 101                 'title': 'Youtube - Dl Test Video \'\' Ä↭', 
 102                 'uploader': 'jaimeMF', 
 103                 'description': 'test chars:  \"\'/\\ä↭', 
 104                 'upload_date': '20131209', 
 106                 'license': 'all-rights-reserved', 
 111             'url': 'https://soundcloud.com/oddsamples/bus-brakes', 
 112             'md5': '7624f2351f8a3b2e7cd51522496e7631', 
 116                 'title': 'Bus Brakes', 
 117                 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66', 
 118                 'uploader': 'oddsamples', 
 119                 'upload_date': '20140109', 
 121                 'license': 'cc-by-sa', 
 124         # private link, downloadable format 
 126             'url': 'https://soundcloud.com/oriuplift/uponly-238-no-talking-wav/s-AyZUd', 
 127             'md5': '64a60b16e617d41d0bef032b7f55441e', 
 131                 'title': 'Uplifting Only 238 [No Talking] (incl. Alex Feed Guestmix) (Aug 31, 2017) [wav]', 
 132                 'description': 'md5:fa20ee0fca76a3d6df8c7e57f3715366', 
 133                 'uploader': 'Ori Uplift Music', 
 134                 'upload_date': '20170831', 
 136                 'license': 'all-rights-reserved', 
 141     _CLIENT_ID 
= 'JlZIsxg2hY5WnBgtn3jfS0UYCl0K8DOg' 
 142     _IPHONE_CLIENT_ID 
= '376f225bf427445fc4bfb6b99b72e0bf' 
 145     def _extract_urls(webpage
): 
 146         return [m
.group('url') for m 
in re
.finditer( 
 147             r
'<iframe[^>]+src=(["\'])(?P
<url
>(?
:https?
://)?
(?
:w\
.)?soundcloud\
.com
/player
.+?
)\
1', 
 150     def report_resolve(self, video_id): 
 151         """Report information extraction.""" 
 152         self.to_screen('%s: Resolving 
id' % video_id) 
 155     def _resolv_url(cls, url): 
 156         return 'https
://api
.soundcloud
.com
/resolve
.json?url
=' + url + '&client_id
=' + cls._CLIENT_ID 
 158     def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None): 
 159         track_id = compat_str(info['id']) 
 160         name = full_title or track_id 
 162             self.report_extraction(name) 
 163         thumbnail = info.get('artwork_url
') 
 164         if isinstance(thumbnail, compat_str): 
 165             thumbnail = thumbnail.replace('-large
', '-t500x500
') 
 169             'uploader
': info.get('user
', {}).get('username
'), 
 170             'upload_date
': unified_strdate(info.get('created_at
')), 
 171             'title
': info['title
'], 
 172             'description
': info.get('description
'), 
 173             'thumbnail
': thumbnail, 
 174             'duration
': int_or_none(info.get('duration
'), 1000), 
 175             'webpage_url
': info.get('permalink_url
'), 
 176             'license
': info.get('license
'), 
 179         query = {'client_id
': self._CLIENT_ID} 
 180         if secret_token is not None: 
 181             query['secret_token
'] = secret_token 
 182         if info.get('downloadable
', False): 
 183             # We can build a direct link to the song 
 184             format_url = update_url_query( 
 185                 'https
://api
.soundcloud
.com
/tracks
/%s/download
' % track_id, query) 
 187                 'format_id
': 'download
', 
 188                 'ext
': info.get('original_format
', 'mp3
'), 
 194         # We have to retrieve the url 
 195         format_dict = self._download_json( 
 196             'https
://api
.soundcloud
.com
/i1
/tracks
/%s/streams
' % track_id, 
 197             track_id, 'Downloading track url
', query=query) 
 199         for key, stream_url in format_dict.items(): 
 200             abr = int_or_none(self._search_regex( 
 201                 r'_(\d
+)_url
', key, 'audio bitrate
', default=None)) 
 202             if key.startswith('http
'): 
 208             elif key.startswith('rtmp
'): 
 209                 # The url doesn't have an rtmp app
, we have to extract the playpath
 
 210                 url
, path 
= stream_url
.split('mp3:', 1) 
 214                     'play_path': 'mp3:' + path
, 
 217             elif key
.startswith('hls'): 
 218                 stream_formats 
= self
._extract
_m
3u8_formats
( 
 219                     stream_url
, track_id
, 'mp3', entry_protocol
='m3u8_native', 
 220                     m3u8_id
=key
, fatal
=False) 
 224             for f 
in stream_formats
: 
 227             formats
.extend(stream_formats
) 
 230             # We fallback to the stream_url in the original info, this 
 231             # cannot be always used, sometimes it can give an HTTP 404 error 
 233                 'format_id': 'fallback', 
 234                 'url': update_url_query(info
['stream_url'], query
), 
 241         self
._check
_formats
(formats
, track_id
) 
 242         self
._sort
_formats
(formats
) 
 243         result
['formats'] = formats
 
 247     def _real_extract(self
, url
): 
 248         mobj 
= re
.match(self
._VALID
_URL
, url
, flags
=re
.VERBOSE
) 
 250             raise ExtractorError('Invalid URL: %s' % url
) 
 252         track_id 
= mobj
.group('track_id') 
 254         if track_id 
is not None: 
 255             info_json_url 
= 'https://api.soundcloud.com/tracks/' + track_id 
+ '.json?client_id=' + self
._CLIENT
_ID
 
 256             full_title 
= track_id
 
 257             token 
= mobj
.group('secret_token') 
 259                 info_json_url 
+= '&secret_token=' + token
 
 260         elif mobj
.group('player'): 
 261             query 
= compat_urlparse
.parse_qs(compat_urlparse
.urlparse(url
).query
) 
 262             real_url 
= query
['url'][0] 
 263             # If the token is in the query of the original url we have to 
 265             if 'secret_token' in query
: 
 266                 real_url 
+= '?secret_token=' + query
['secret_token'][0] 
 267             return self
.url_result(real_url
) 
 269             # extract uploader (which is in the url) 
 270             uploader 
= mobj
.group('uploader') 
 271             # extract simple title (uploader + slug of song title) 
 272             slug_title 
= mobj
.group('title') 
 273             token 
= mobj
.group('token') 
 274             full_title 
= resolve_title 
= '%s/%s' % (uploader
, slug_title
) 
 276                 resolve_title 
+= '/%s' % token
 
 278             self
.report_resolve(full_title
) 
 280             url 
= 'https://soundcloud.com/%s' % resolve_title
 
 281             info_json_url 
= self
._resolv
_url
(url
) 
 282         info 
= self
._download
_json
(info_json_url
, full_title
, 'Downloading info JSON') 
 284         return self
._extract
_info
_dict
(info
, full_title
, secret_token
=token
) 
 287 class SoundcloudPlaylistBaseIE(SoundcloudIE
): 
 290         return compat_str(e
['id']) if e
.get('id') else None 
 292     def _extract_track_entries(self
, tracks
): 
 295                 track
['permalink_url'], SoundcloudIE
.ie_key(), 
 296                 video_id
=self
._extract
_id
(track
)) 
 297             for track 
in tracks 
if track
.get('permalink_url')] 
 300 class SoundcloudSetIE(SoundcloudPlaylistBaseIE
): 
 301     _VALID_URL 
= r
'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?' 
 302     IE_NAME 
= 'soundcloud:set' 
 304         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep', 
 307             'title': 'The Royal Concept EP', 
 309         'playlist_mincount': 5, 
 311         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token', 
 312         'only_matching': True, 
 315     def _real_extract(self
, url
): 
 316         mobj 
= re
.match(self
._VALID
_URL
, url
) 
 318         # extract uploader (which is in the url) 
 319         uploader 
= mobj
.group('uploader') 
 320         # extract simple title (uploader + slug of song title) 
 321         slug_title 
= mobj
.group('slug_title') 
 322         full_title 
= '%s/sets/%s' % (uploader
, slug_title
) 
 323         url 
= 'https://soundcloud.com/%s/sets/%s' % (uploader
, slug_title
) 
 325         token 
= mobj
.group('token') 
 327             full_title 
+= '/' + token
 
 330         self
.report_resolve(full_title
) 
 332         resolv_url 
= self
._resolv
_url
(url
) 
 333         info 
= self
._download
_json
(resolv_url
, full_title
) 
 336             msgs 
= (compat_str(err
['error_message']) for err 
in info
['errors']) 
 337             raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs
)) 
 339         entries 
= self
._extract
_track
_entries
(info
['tracks']) 
 344             'id': '%s' % info
['id'], 
 345             'title': info
['title'], 
 349 class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE
): 
 350     _API_BASE 
= 'https://api.soundcloud.com' 
 351     _API_V2_BASE 
= 'https://api-v2.soundcloud.com' 
 353     def _extract_playlist(self
, base_url
, playlist_id
, playlist_title
): 
 356             'client_id': self
._CLIENT
_ID
, 
 357             'linked_partitioning': '1', 
 360         query 
= COMMON_QUERY
.copy() 
 363         next_href 
= base_url 
+ '?' + compat_urllib_parse_urlencode(query
) 
 366         for i 
in itertools
.count(): 
 367             response 
= self
._download
_json
( 
 368                 next_href
, playlist_id
, 'Downloading track page %s' % (i 
+ 1)) 
 370             collection 
= response
['collection'] 
 374             def resolve_permalink_url(candidates
): 
 375                 for cand 
in candidates
: 
 376                     if isinstance(cand
, dict): 
 377                         permalink_url 
= cand
.get('permalink_url') 
 378                         entry_id 
= self
._extract
_id
(cand
) 
 379                         if permalink_url 
and permalink_url
.startswith('http'): 
 380                             return permalink_url
, entry_id
 
 383                 permalink_url
, entry_id 
= resolve_permalink_url((e
, e
.get('track'), e
.get('playlist'))) 
 385                     entries
.append(self
.url_result(permalink_url
, video_id
=entry_id
)) 
 387             next_href 
= response
.get('next_href') 
 391             parsed_next_href 
= compat_urlparse
.urlparse(response
['next_href']) 
 392             qs 
= compat_urlparse
.parse_qs(parsed_next_href
.query
) 
 393             qs
.update(COMMON_QUERY
) 
 394             next_href 
= compat_urlparse
.urlunparse( 
 395                 parsed_next_href
._replace
(query
=compat_urllib_parse_urlencode(qs
, True))) 
 400             'title': playlist_title
, 
 405 class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE
): 
 406     _VALID_URL 
= r
'''(?x) 
 408                             (?:(?:www|m)\.)?soundcloud\.com/ 
 411                                 (?P<rsrc>tracks|sets|reposts|likes|spotlight) 
 415     IE_NAME 
= 'soundcloud:user' 
 417         'url': 'https://soundcloud.com/the-akashic-chronicler', 
 420             'title': 'The Akashic Chronicler (All)', 
 422         'playlist_mincount': 74, 
 424         'url': 'https://soundcloud.com/the-akashic-chronicler/tracks', 
 427             'title': 'The Akashic Chronicler (Tracks)', 
 429         'playlist_mincount': 37, 
 431         'url': 'https://soundcloud.com/the-akashic-chronicler/sets', 
 434             'title': 'The Akashic Chronicler (Playlists)', 
 436         'playlist_mincount': 2, 
 438         'url': 'https://soundcloud.com/the-akashic-chronicler/reposts', 
 441             'title': 'The Akashic Chronicler (Reposts)', 
 443         'playlist_mincount': 7, 
 445         'url': 'https://soundcloud.com/the-akashic-chronicler/likes', 
 448             'title': 'The Akashic Chronicler (Likes)', 
 450         'playlist_mincount': 321, 
 452         'url': 'https://soundcloud.com/grynpyret/spotlight', 
 455             'title': 'Grynpyret (Spotlight)', 
 457         'playlist_mincount': 1, 
 461         'all': '%s/profile/soundcloud:users:%%s' % SoundcloudPagedPlaylistBaseIE
._API
_V
2_BASE
, 
 462         'tracks': '%s/users/%%s/tracks' % SoundcloudPagedPlaylistBaseIE
._API
_BASE
, 
 463         'sets': '%s/users/%%s/playlists' % SoundcloudPagedPlaylistBaseIE
._API
_V
2_BASE
, 
 464         'reposts': '%s/profile/soundcloud:users:%%s/reposts' % SoundcloudPagedPlaylistBaseIE
._API
_V
2_BASE
, 
 465         'likes': '%s/users/%%s/likes' % SoundcloudPagedPlaylistBaseIE
._API
_V
2_BASE
, 
 466         'spotlight': '%s/users/%%s/spotlight' % SoundcloudPagedPlaylistBaseIE
._API
_V
2_BASE
, 
 473         'reposts': 'Reposts', 
 475         'spotlight': 'Spotlight', 
 478     def _real_extract(self
, url
): 
 479         mobj 
= re
.match(self
._VALID
_URL
, url
) 
 480         uploader 
= mobj
.group('user') 
 482         url 
= 'https://soundcloud.com/%s/' % uploader
 
 483         resolv_url 
= self
._resolv
_url
(url
) 
 484         user 
= self
._download
_json
( 
 485             resolv_url
, uploader
, 'Downloading user info') 
 487         resource 
= mobj
.group('rsrc') or 'all' 
 489         return self
._extract
_playlist
( 
 490             self
._BASE
_URL
_MAP
[resource
] % user
['id'], compat_str(user
['id']), 
 491             '%s (%s)' % (user
['username'], self
._TITLE
_MAP
[resource
])) 
 494 class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE
): 
 495     _VALID_URL 
= r
'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)' 
 496     IE_NAME 
= 'soundcloud:trackstation' 
 498         'url': 'https://soundcloud.com/stations/track/officialsundial/your-text', 
 501             'title': 'Track station: your-text', 
 503         'playlist_mincount': 47, 
 506     def _real_extract(self
, url
): 
 507         track_name 
= self
._match
_id
(url
) 
 509         webpage 
= self
._download
_webpage
(url
, track_name
) 
 511         track_id 
= self
._search
_regex
( 
 512             r
'soundcloud:track-stations:(\d+)', webpage
, 'track id') 
 514         return self
._extract
_playlist
( 
 515             '%s/stations/soundcloud:track-stations:%s/tracks' 
 516             % (self
._API
_V
2_BASE
, track_id
), 
 517             track_id
, 'Track station: %s' % track_name
) 
 520 class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE
): 
 521     _VALID_URL 
= r
'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$' 
 522     IE_NAME 
= 'soundcloud:playlist' 
 524         'url': 'https://api.soundcloud.com/playlists/4110309', 
 527             'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]', 
 528             'description': 're:.*?TILT Brass - Bowery Poetry Club', 
 533     def _real_extract(self
, url
): 
 534         mobj 
= re
.match(self
._VALID
_URL
, url
) 
 535         playlist_id 
= mobj
.group('id') 
 536         base_url 
= '%s//api.soundcloud.com/playlists/%s.json?' % (self
.http_scheme(), playlist_id
) 
 539             'client_id': self
._CLIENT
_ID
, 
 541         token 
= mobj
.group('token') 
 544             data_dict
['secret_token'] = token
 
 546         data 
= compat_urllib_parse_urlencode(data_dict
) 
 547         data 
= self
._download
_json
( 
 548             base_url 
+ data
, playlist_id
, 'Downloading playlist') 
 550         entries 
= self
._extract
_track
_entries
(data
['tracks']) 
 555             'title': data
.get('title'), 
 556             'description': data
.get('description'), 
 561 class SoundcloudSearchIE(SearchInfoExtractor
, SoundcloudIE
): 
 562     IE_NAME 
= 'soundcloud:search' 
 563     IE_DESC 
= 'Soundcloud search' 
 564     _MAX_RESULTS 
= float('inf') 
 566         'url': 'scsearch15:post-avant jazzcore', 
 568             'title': 'post-avant jazzcore', 
 570         'playlist_count': 15, 
 573     _SEARCH_KEY 
= 'scsearch' 
 574     _MAX_RESULTS_PER_PAGE 
= 200 
 575     _DEFAULT_RESULTS_PER_PAGE 
= 50 
 576     _API_V2_BASE 
= 'https://api-v2.soundcloud.com' 
 578     def _get_collection(self
, endpoint
, collection_id
, **query
): 
 580             query
.get('limit', self
._DEFAULT
_RESULTS
_PER
_PAGE
), 
 581             self
._MAX
_RESULTS
_PER
_PAGE
) 
 582         query
['limit'] = limit
 
 583         query
['client_id'] = self
._CLIENT
_ID
 
 584         query
['linked_partitioning'] = '1' 
 586         data 
= compat_urllib_parse_urlencode(query
) 
 587         next_url 
= '{0}{1}?{2}'.format(self
._API
_V
2_BASE
, endpoint
, data
) 
 589         collected_results 
= 0 
 591         for i 
in itertools
.count(1): 
 592             response 
= self
._download
_json
( 
 593                 next_url
, collection_id
, 'Downloading page {0}'.format(i
), 
 594                 'Unable to download API page') 
 596             collection 
= response
.get('collection', []) 
 600             collection 
= list(filter(bool, collection
)) 
 601             collected_results 
+= len(collection
) 
 603             for item 
in collection
: 
 604                 yield self
.url_result(item
['uri'], SoundcloudIE
.ie_key()) 
 606             if not collection 
or collected_results 
>= limit
: 
 609             next_url 
= response
.get('next_href') 
 613     def _get_n_results(self
, query
, n
): 
 614         tracks 
= self
._get
_collection
('/search/tracks', query
, limit
=n
, q
=query
) 
 615         return self
.playlist_result(tracks
, playlist_title
=query
)