]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/yandexmusic.py
Merge tag 'upstream/2016.06.25'
[youtubedl] / youtube_dl / extractor / yandexmusic.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5 import hashlib
6
7 from .common import InfoExtractor
8 from ..compat import compat_str
9 from ..utils import (
10 ExtractorError,
11 int_or_none,
12 float_or_none,
13 )
14
15
16 class YandexMusicBaseIE(InfoExtractor):
17 @staticmethod
18 def _handle_error(response):
19 if isinstance(response, dict):
20 error = response.get('error')
21 if error:
22 raise ExtractorError(error, expected=True)
23 if response.get('type') == 'captcha' or 'captcha' in response:
24 YandexMusicBaseIE._raise_captcha()
25
26 @staticmethod
27 def _raise_captcha():
28 raise ExtractorError(
29 'YandexMusic has considered youtube-dl requests automated and '
30 'asks you to solve a CAPTCHA. You can either wait for some '
31 'time until unblocked and optionally use --sleep-interval '
32 'in future or alternatively you can go to https://music.yandex.ru/ '
33 'solve CAPTCHA, then export cookies and pass cookie file to '
34 'youtube-dl with --cookies',
35 expected=True)
36
37 def _download_webpage(self, *args, **kwargs):
38 webpage = super(YandexMusicBaseIE, self)._download_webpage(*args, **kwargs)
39 if 'Нам очень жаль, но запросы, поступившие с вашего IP-адреса, похожи на автоматические.' in webpage:
40 self._raise_captcha()
41 return webpage
42
43 def _download_json(self, *args, **kwargs):
44 response = super(YandexMusicBaseIE, self)._download_json(*args, **kwargs)
45 self._handle_error(response)
46 return response
47
48
49 class YandexMusicTrackIE(YandexMusicBaseIE):
50 IE_NAME = 'yandexmusic:track'
51 IE_DESC = 'Яндекс.Музыка - Трек'
52 _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
53
54 _TEST = {
55 'url': 'http://music.yandex.ru/album/540508/track/4878838',
56 'md5': 'f496818aa2f60b6c0062980d2e00dc20',
57 'info_dict': {
58 'id': '4878838',
59 'ext': 'mp3',
60 'title': 'Carlo Ambrosio & Fabio Di Bari, Carlo Ambrosio - Gypsy Eyes 1',
61 'filesize': 4628061,
62 'duration': 193.04,
63 'track': 'Gypsy Eyes 1',
64 'album': 'Gypsy Soul',
65 'album_artist': 'Carlo Ambrosio',
66 'artist': 'Carlo Ambrosio & Fabio Di Bari, Carlo Ambrosio',
67 'release_year': '2009',
68 },
69 'skip': 'Travis CI servers blocked by YandexMusic',
70 }
71
72 def _get_track_url(self, storage_dir, track_id):
73 data = self._download_json(
74 'http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?action=getTrackSrc&p=download-info/%s'
75 % storage_dir,
76 track_id, 'Downloading track location JSON')
77
78 key = hashlib.md5(('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']).encode('utf-8')).hexdigest()
79 storage = storage_dir.split('.')
80
81 return ('http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default'
82 % (data['host'], key, data['ts'] + data['path'], storage[1]))
83
84 def _get_track_info(self, track):
85 thumbnail = None
86 cover_uri = track.get('albums', [{}])[0].get('coverUri')
87 if cover_uri:
88 thumbnail = cover_uri.replace('%%', 'orig')
89 if not thumbnail.startswith('http'):
90 thumbnail = 'http://' + thumbnail
91
92 track_title = track['title']
93 track_info = {
94 'id': track['id'],
95 'ext': 'mp3',
96 'url': self._get_track_url(track['storageDir'], track['id']),
97 'filesize': int_or_none(track.get('fileSize')),
98 'duration': float_or_none(track.get('durationMs'), 1000),
99 'thumbnail': thumbnail,
100 'track': track_title,
101 }
102
103 def extract_artist(artist_list):
104 if artist_list and isinstance(artist_list, list):
105 artists_names = [a['name'] for a in artist_list if a.get('name')]
106 if artists_names:
107 return ', '.join(artists_names)
108
109 albums = track.get('albums')
110 if albums and isinstance(albums, list):
111 album = albums[0]
112 if isinstance(album, dict):
113 year = album.get('year')
114 track_info.update({
115 'album': album.get('title'),
116 'album_artist': extract_artist(album.get('artists')),
117 'release_year': compat_str(year) if year else None,
118 })
119
120 track_artist = extract_artist(track.get('artists'))
121 if track_artist:
122 track_info.update({
123 'artist': track_artist,
124 'title': '%s - %s' % (track_artist, track_title),
125 })
126 else:
127 track_info['title'] = track_title
128 return track_info
129
130 def _real_extract(self, url):
131 mobj = re.match(self._VALID_URL, url)
132 album_id, track_id = mobj.group('album_id'), mobj.group('id')
133
134 track = self._download_json(
135 'http://music.yandex.ru/handlers/track.jsx?track=%s:%s' % (track_id, album_id),
136 track_id, 'Downloading track JSON')['track']
137
138 return self._get_track_info(track)
139
140
141 class YandexMusicPlaylistBaseIE(YandexMusicBaseIE):
142 def _build_playlist(self, tracks):
143 return [
144 self.url_result(
145 'http://music.yandex.ru/album/%s/track/%s' % (track['albums'][0]['id'], track['id']))
146 for track in tracks if track.get('albums') and isinstance(track.get('albums'), list)]
147
148
149 class YandexMusicAlbumIE(YandexMusicPlaylistBaseIE):
150 IE_NAME = 'yandexmusic:album'
151 IE_DESC = 'Яндекс.Музыка - Альбом'
152 _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<id>\d+)/?(\?|$)'
153
154 _TEST = {
155 'url': 'http://music.yandex.ru/album/540508',
156 'info_dict': {
157 'id': '540508',
158 'title': 'Carlo Ambrosio - Gypsy Soul (2009)',
159 },
160 'playlist_count': 50,
161 'skip': 'Travis CI servers blocked by YandexMusic',
162 }
163
164 def _real_extract(self, url):
165 album_id = self._match_id(url)
166
167 album = self._download_json(
168 'http://music.yandex.ru/handlers/album.jsx?album=%s' % album_id,
169 album_id, 'Downloading album JSON')
170
171 entries = self._build_playlist(album['volumes'][0])
172
173 title = '%s - %s' % (album['artists'][0]['name'], album['title'])
174 year = album.get('year')
175 if year:
176 title += ' (%s)' % year
177
178 return self.playlist_result(entries, compat_str(album['id']), title)
179
180
181 class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
182 IE_NAME = 'yandexmusic:playlist'
183 IE_DESC = 'Яндекс.Музыка - Плейлист'
184 _VALID_URL = r'https?://music\.yandex\.(?P<tld>ru|kz|ua|by)/users/(?P<user>[^/]+)/playlists/(?P<id>\d+)'
185
186 _TESTS = [{
187 'url': 'http://music.yandex.ru/users/music.partners/playlists/1245',
188 'info_dict': {
189 'id': '1245',
190 'title': 'Что слушают Enter Shikari',
191 'description': 'md5:3b9f27b0efbe53f2ee1e844d07155cc9',
192 },
193 'playlist_count': 6,
194 'skip': 'Travis CI servers blocked by YandexMusic',
195 }, {
196 # playlist exceeding the limit of 150 tracks shipped with webpage (see
197 # https://github.com/rg3/youtube-dl/issues/6666)
198 'url': 'https://music.yandex.ru/users/ya.playlist/playlists/1036',
199 'info_dict': {
200 'id': '1036',
201 'title': 'Музыка 90-х',
202 },
203 'playlist_mincount': 300,
204 'skip': 'Travis CI servers blocked by YandexMusic',
205 }]
206
207 def _real_extract(self, url):
208 mobj = re.match(self._VALID_URL, url)
209 tld = mobj.group('tld')
210 user = mobj.group('user')
211 playlist_id = mobj.group('id')
212
213 playlist = self._download_json(
214 'https://music.yandex.%s/handlers/playlist.jsx' % tld,
215 playlist_id, 'Downloading missing tracks JSON',
216 fatal=False,
217 headers={
218 'Referer': url,
219 'X-Requested-With': 'XMLHttpRequest',
220 'X-Retpath-Y': url,
221 },
222 query={
223 'owner': user,
224 'kinds': playlist_id,
225 'light': 'true',
226 'lang': tld,
227 'external-domain': 'music.yandex.%s' % tld,
228 'overembed': 'false',
229 })['playlist']
230
231 tracks, track_ids = playlist['tracks'], map(compat_str, playlist['trackIds'])
232
233 # tracks dictionary shipped with playlist.jsx API is limited to 150 tracks,
234 # missing tracks should be retrieved manually.
235 if len(tracks) < len(track_ids):
236 present_track_ids = set([
237 compat_str(track['id'])
238 for track in tracks if track.get('id')])
239 missing_track_ids = [
240 track_id for track_id in track_ids
241 if track_id not in present_track_ids]
242 missing_tracks = self._download_json(
243 'https://music.yandex.%s/handlers/track-entries.jsx' % tld,
244 playlist_id, 'Downloading missing tracks JSON',
245 fatal=False,
246 headers={
247 'Referer': url,
248 'X-Requested-With': 'XMLHttpRequest',
249 },
250 query={
251 'entries': ','.join(missing_track_ids),
252 'lang': tld,
253 'external-domain': 'music.yandex.%s' % tld,
254 'overembed': 'false',
255 'strict': 'true',
256 })
257 if missing_tracks:
258 tracks.extend(missing_tracks)
259
260 return self.playlist_result(
261 self._build_playlist(tracks),
262 compat_str(playlist_id),
263 playlist.get('title'), playlist.get('description'))