Merge pull request #4 from debian-janitor/lintian-fixes

[youtubedl] / youtube_dl / extractor / soundcloud.py

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index a0b09f5b1747e3b1b9b3a17b10d482b47512143a..ff6be0b545a79260341ec25e6059d306fe859ef8 100644 (file)
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -27,6 +27,7 @@ from ..utils import (
     unified_timestamp,
     update_url_query,
     url_or_none,
     unified_timestamp,
     update_url_query,
     url_or_none,

+    urlhandle_detect_ext,

 )
 
 
 )
 
 


@@ -96,7 +97,7 @@ class SoundcloudIE(InfoExtractor):
                 'repost_count': int,
             }
         },
                 'repost_count': int,
             }
         },

-        # not streamable song, preview
+        # geo-restricted

         {
             'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
             'info_dict': {
         {
             'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
             'info_dict': {


@@ -108,17 +109,13 @@ class SoundcloudIE(InfoExtractor):
                 'uploader_id': '9615865',
                 'timestamp': 1337635207,
                 'upload_date': '20120521',
                 'uploader_id': '9615865',
                 'timestamp': 1337635207,
                 'upload_date': '20120521',

-                'duration': 30,
+                'duration': 227.155,

                 'license': 'all-rights-reserved',
                 'view_count': int,
                 'like_count': int,
                 'comment_count': int,
                 'repost_count': int,
             },
                 'license': 'all-rights-reserved',
                 'view_count': int,
                 'like_count': int,
                 'comment_count': int,
                 'repost_count': int,
             },

-            'params': {
-                # rtmp
-                'skip_download': True,
-            },

         },
         # private link
         {
         },
         # private link
         {


@@ -229,7 +226,6 @@ class SoundcloudIE(InfoExtractor):
                 'skip_download': True,
             },
         },
                 'skip_download': True,
             },
         },

-        # not available via api.soundcloud.com/i1/tracks/id/streams

         {
             'url': 'https://soundcloud.com/giovannisarani/mezzo-valzer',
             'md5': 'e22aecd2bc88e0e4e432d7dcc0a1abf7',
         {
             'url': 'https://soundcloud.com/giovannisarani/mezzo-valzer',
             'md5': 'e22aecd2bc88e0e4e432d7dcc0a1abf7',


@@ -238,7 +234,7 @@ class SoundcloudIE(InfoExtractor):
                 'ext': 'mp3',
                 'title': 'Mezzo Valzer',
                 'description': 'md5:4138d582f81866a530317bae316e8b61',
                 'ext': 'mp3',
                 'title': 'Mezzo Valzer',
                 'description': 'md5:4138d582f81866a530317bae316e8b61',

-                'uploader': 'Giovanni Sarani',
+                'uploader': 'Micronie',

                 'uploader_id': '3352531',
                 'timestamp': 1551394171,
                 'upload_date': '20190228',
                 'uploader_id': '3352531',
                 'timestamp': 1551394171,
                 'upload_date': '20190228',


@@ -250,11 +246,9 @@ class SoundcloudIE(InfoExtractor):
                 'comment_count': int,
                 'repost_count': int,
             },
                 'comment_count': int,
                 'repost_count': int,
             },

-            'expected_warnings': ['Unable to download JSON metadata'],

         }
     ]
 
         }
     ]
 

-    _API_BASE = 'https://api.soundcloud.com/'

     _API_V2_BASE = 'https://api-v2.soundcloud.com/'
     _BASE_URL = 'https://soundcloud.com/'
     _IMAGE_REPL_RE = r'-([0-9a-z]+)\.jpg'
     _API_V2_BASE = 'https://api-v2.soundcloud.com/'
     _BASE_URL = 'https://soundcloud.com/'
     _IMAGE_REPL_RE = r'-([0-9a-z]+)\.jpg'


@@ -316,10 +310,9 @@ class SoundcloudIE(InfoExtractor):
     def _resolv_url(cls, url):
         return SoundcloudIE._API_V2_BASE + 'resolve?url=' + url
 
     def _resolv_url(cls, url):
         return SoundcloudIE._API_V2_BASE + 'resolve?url=' + url
 

-    def _extract_info_dict(self, info, full_title=None, secret_token=None, version=2):
+    def _extract_info_dict(self, info, full_title=None, secret_token=None):

         track_id = compat_str(info['id'])
         title = info['title']
         track_id = compat_str(info['id'])
         title = info['title']

-        track_base_url = self._API_BASE + 'tracks/%s' % track_id

 
         format_urls = set()
         formats = []
 
         format_urls = set()
         formats = []


@@ -328,21 +321,22 @@ class SoundcloudIE(InfoExtractor):
             query['secret_token'] = secret_token
 
         if info.get('downloadable') and info.get('has_downloads_left'):
             query['secret_token'] = secret_token
 
         if info.get('downloadable') and info.get('has_downloads_left'):

-            format_url = update_url_query(
-                info.get('download_url') or track_base_url + '/download', query)
-            format_urls.add(format_url)
-            if version == 2:
-                v1_info = self._download_json(
-                    track_base_url, track_id, query=query, fatal=False) or {}
-            else:
-                v1_info = info
-            formats.append({
-                'format_id': 'download',
-                'ext': v1_info.get('original_format') or 'mp3',
-                'filesize': int_or_none(v1_info.get('original_content_size')),
-                'url': format_url,
-                'preference': 10,
-            })
+            download_url = update_url_query(
+                self._API_V2_BASE + 'tracks/' + track_id + '/download', query)
+            redirect_url = (self._download_json(download_url, track_id, fatal=False) or {}).get('redirectUri')
+            if redirect_url:
+                urlh = self._request_webpage(
+                    HEADRequest(redirect_url), track_id, fatal=False)
+                if urlh:
+                    format_url = urlh.geturl()
+                    format_urls.add(format_url)
+                    formats.append({
+                        'format_id': 'download',
+                        'ext': urlhandle_detect_ext(urlh) or 'mp3',
+                        'filesize': int_or_none(urlh.headers.get('Content-Length')),
+                        'url': format_url,
+                        'preference': 10,
+                    })

 
         def invalid_url(url):
             return not url or url in format_urls
 
         def invalid_url(url):
             return not url or url in format_urls


@@ -406,42 +400,11 @@ class SoundcloudIE(InfoExtractor):
             }, 'http' if protocol == 'progressive' else protocol,
                 t.get('snipped') or '/preview/' in format_url)
 
             }, 'http' if protocol == 'progressive' else protocol,
                 t.get('snipped') or '/preview/' in format_url)
 

-        if not formats:
-            # Old API, does not work for some tracks (e.g.
-            # https://soundcloud.com/giovannisarani/mezzo-valzer)
-            # and might serve preview URLs (e.g.
-            # http://www.soundcloud.com/snbrn/ele)
-            format_dict = self._download_json(
-                track_base_url + '/streams', track_id,
-                'Downloading track url', query=query, fatal=False) or {}
-
-            for key, stream_url in format_dict.items():
-                if invalid_url(stream_url):
-                    continue
-                format_urls.add(stream_url)
-                mobj = re.search(r'(http|hls)_([^_]+)_(\d+)_url', key)
-                if mobj:
-                    protocol, ext, abr = mobj.groups()
-                    add_format({
-                        'abr': abr,
-                        'ext': ext,
-                        'url': stream_url,
-                    }, protocol)
-
-        if not formats:
-            # We fallback to the stream_url in the original info, this
-            # cannot be always used, sometimes it can give an HTTP 404 error
-            urlh = self._request_webpage(
-                HEADRequest(info.get('stream_url') or track_base_url + '/stream'),
-                track_id, query=query, fatal=False)
-            if urlh:
-                stream_url = urlh.geturl()
-                if not invalid_url(stream_url):
-                    add_format({'url': stream_url}, 'http')
-

         for f in formats:
             f['vcodec'] = 'none'
 
         for f in formats:
             f['vcodec'] = 'none'
 

+        if not formats and info.get('policy') == 'BLOCK':
+            self.raise_geo_restricted()

         self._sort_formats(formats)
 
         user = info.get('user') or {}
         self._sort_formats(formats)
 
         user = info.get('user') or {}


@@ -511,20 +474,24 @@ class SoundcloudIE(InfoExtractor):
                 resolve_title += '/%s' % token
             info_json_url = self._resolv_url(self._BASE_URL + resolve_title)
 
                 resolve_title += '/%s' % token
             info_json_url = self._resolv_url(self._BASE_URL + resolve_title)
 

-        version = 2

         info = self._download_json(
         info = self._download_json(

-            info_json_url, full_title, 'Downloading info JSON', query=query, fatal=False)
-        if not info:
-            info = self._download_json(
-                info_json_url.replace(self._API_V2_BASE, self._API_BASE),
-                full_title, 'Downloading info JSON', query=query)
-            version = 1
+            info_json_url, full_title, 'Downloading info JSON', query=query)

 
 

-        return self._extract_info_dict(info, full_title, token, version)
+        return self._extract_info_dict(info, full_title, token)

 
 
 class SoundcloudPlaylistBaseIE(SoundcloudIE):
 
 
 class SoundcloudPlaylistBaseIE(SoundcloudIE):

-    def _extract_track_entries(self, tracks, token=None):
+    def _extract_set(self, playlist, token=None):
+        playlist_id = compat_str(playlist['id'])
+        tracks = playlist.get('tracks') or []
+        if not all([t.get('permalink_url') for t in tracks]) and token:
+            tracks = self._download_json(
+                self._API_V2_BASE + 'tracks', playlist_id,
+                'Downloading tracks', query={
+                    'ids': ','.join([compat_str(t['id']) for t in tracks]),
+                    'playlistId': playlist_id,
+                    'playlistSecretToken': token,
+                })

         entries = []
         for track in tracks:
             track_id = str_or_none(track.get('id'))
         entries = []
         for track in tracks:
             track_id = str_or_none(track.get('id'))


@@ -537,7 +504,10 @@ class SoundcloudPlaylistBaseIE(SoundcloudIE):
                     url += '?secret_token=' + token
             entries.append(self.url_result(
                 url, SoundcloudIE.ie_key(), track_id))
                     url += '?secret_token=' + token
             entries.append(self.url_result(
                 url, SoundcloudIE.ie_key(), track_id))

-        return entries
+        return self.playlist_result(
+            entries, playlist_id,
+            playlist.get('title'),
+            playlist.get('description'))

 
 
 class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
 
 
 class SoundcloudSetIE(SoundcloudPlaylistBaseIE):


@@ -548,6 +518,7 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
         'info_dict': {
             'id': '2284613',
             'title': 'The Royal Concept EP',
         'info_dict': {
             'id': '2284613',
             'title': 'The Royal Concept EP',

+            'description': 'md5:71d07087c7a449e8941a70a29e34671e',

         },
         'playlist_mincount': 5,
     }, {
         },
         'playlist_mincount': 5,
     }, {


@@ -570,13 +541,10 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
             msgs = (compat_str(err['error_message']) for err in info['errors'])
             raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 
             msgs = (compat_str(err['error_message']) for err in info['errors'])
             raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 

-        entries = self._extract_track_entries(info['tracks'], token)
-
-        return self.playlist_result(
-            entries, str_or_none(info.get('id')), info.get('title'))
+        return self._extract_set(info, token)

 
 
 
 

-class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
+class SoundcloudPagedPlaylistBaseIE(SoundcloudIE):

     def _extract_playlist(self, base_url, playlist_id, playlist_title):
         COMMON_QUERY = {
             'limit': 2000000000,
     def _extract_playlist(self, base_url, playlist_id, playlist_title):
         COMMON_QUERY = {
             'limit': 2000000000,


@@ -774,10 +742,7 @@ class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
             self._API_V2_BASE + 'playlists/' + playlist_id,
             playlist_id, 'Downloading playlist', query=query)
 
             self._API_V2_BASE + 'playlists/' + playlist_id,
             playlist_id, 'Downloading playlist', query=query)
 

-        entries = self._extract_track_entries(data['tracks'], token)
-
-        return self.playlist_result(
-            entries, playlist_id, data.get('title'), data.get('description'))
+        return self._extract_set(data, token)

 
 
 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
 
 
 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):