]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/common.py
Prepare for release.
[youtubedl] / youtube_dl / extractor / common.py
1 import base64
2 import os
3 import re
4 import socket
5 import sys
6
7 from ..utils import (
8 compat_http_client,
9 compat_urllib_error,
10 compat_urllib_request,
11 compat_str,
12
13 clean_html,
14 compiled_regex_type,
15 ExtractorError,
16 )
17
18 class InfoExtractor(object):
19 """Information Extractor class.
20
21 Information extractors are the classes that, given a URL, extract
22 information about the video (or videos) the URL refers to. This
23 information includes the real video URL, the video title, author and
24 others. The information is stored in a dictionary which is then
25 passed to the FileDownloader. The FileDownloader processes this
26 information possibly downloading the video to the file system, among
27 other possible outcomes.
28
29 The dictionaries must include the following fields:
30
31 id: Video identifier.
32 url: Final video URL.
33 title: Video title, unescaped.
34 ext: Video filename extension.
35
36 The following fields are optional:
37
38 format: The video format, defaults to ext (used for --get-format)
39 thumbnail: Full URL to a video thumbnail image.
40 description: One-line video description.
41 uploader: Full name of the video uploader.
42 upload_date: Video upload date (YYYYMMDD).
43 uploader_id: Nickname or id of the video uploader.
44 location: Physical location of the video.
45 player_url: SWF Player URL (used for rtmpdump).
46 subtitles: The subtitle file contents.
47 view_count: How many users have watched the video on the platform.
48 urlhandle: [internal] The urlHandle to be used to download the file,
49 like returned by urllib.request.urlopen
50
51 The fields should all be Unicode strings.
52
53 Subclasses of this one should re-define the _real_initialize() and
54 _real_extract() methods and define a _VALID_URL regexp.
55 Probably, they should also be added to the list of extractors.
56
57 _real_extract() must return a *list* of information dictionaries as
58 described above.
59
60 Finally, the _WORKING attribute should be set to False for broken IEs
61 in order to warn the users and skip the tests.
62 """
63
64 _ready = False
65 _downloader = None
66 _WORKING = True
67
68 def __init__(self, downloader=None):
69 """Constructor. Receives an optional downloader."""
70 self._ready = False
71 self.set_downloader(downloader)
72
73 @classmethod
74 def suitable(cls, url):
75 """Receives a URL and returns True if suitable for this IE."""
76 return re.match(cls._VALID_URL, url) is not None
77
78 @classmethod
79 def working(cls):
80 """Getter method for _WORKING."""
81 return cls._WORKING
82
83 def initialize(self):
84 """Initializes an instance (authentication, etc)."""
85 if not self._ready:
86 self._real_initialize()
87 self._ready = True
88
89 def extract(self, url):
90 """Extracts URL information and returns it in list of dicts."""
91 self.initialize()
92 return self._real_extract(url)
93
94 def set_downloader(self, downloader):
95 """Sets the downloader for this IE."""
96 self._downloader = downloader
97
98 def _real_initialize(self):
99 """Real initialization process. Redefine in subclasses."""
100 pass
101
102 def _real_extract(self, url):
103 """Real extraction process. Redefine in subclasses."""
104 pass
105
106 @property
107 def IE_NAME(self):
108 return type(self).__name__[:-2]
109
110 def _request_webpage(self, url_or_request, video_id, note=None, errnote=None):
111 """ Returns the response handle """
112 if note is None:
113 self.report_download_webpage(video_id)
114 elif note is not False:
115 self.to_screen(u'%s: %s' % (video_id, note))
116 try:
117 return compat_urllib_request.urlopen(url_or_request)
118 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
119 if errnote is None:
120 errnote = u'Unable to download webpage'
121 raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2])
122
123 def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None):
124 """ Returns a tuple (page content as string, URL handle) """
125 urlh = self._request_webpage(url_or_request, video_id, note, errnote)
126 content_type = urlh.headers.get('Content-Type', '')
127 m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
128 if m:
129 encoding = m.group(1)
130 else:
131 encoding = 'utf-8'
132 webpage_bytes = urlh.read()
133 if self._downloader.params.get('dump_intermediate_pages', False):
134 try:
135 url = url_or_request.get_full_url()
136 except AttributeError:
137 url = url_or_request
138 self.to_screen(u'Dumping request to ' + url)
139 dump = base64.b64encode(webpage_bytes).decode('ascii')
140 self._downloader.to_screen(dump)
141 content = webpage_bytes.decode(encoding, 'replace')
142 return (content, urlh)
143
144 def _download_webpage(self, url_or_request, video_id, note=None, errnote=None):
145 """ Returns the data of the page as a string """
146 return self._download_webpage_handle(url_or_request, video_id, note, errnote)[0]
147
148 def to_screen(self, msg):
149 """Print msg to screen, prefixing it with '[ie_name]'"""
150 self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg))
151
152 def report_extraction(self, id_or_name):
153 """Report information extraction."""
154 self.to_screen(u'%s: Extracting information' % id_or_name)
155
156 def report_download_webpage(self, video_id):
157 """Report webpage download."""
158 self.to_screen(u'%s: Downloading webpage' % video_id)
159
160 def report_age_confirmation(self):
161 """Report attempt to confirm age."""
162 self.to_screen(u'Confirming age')
163
164 #Methods for following #608
165 #They set the correct value of the '_type' key
166 def video_result(self, video_info):
167 """Returns a video"""
168 video_info['_type'] = 'video'
169 return video_info
170 def url_result(self, url, ie=None):
171 """Returns a url that points to a page that should be processed"""
172 #TODO: ie should be the class used for getting the info
173 video_info = {'_type': 'url',
174 'url': url,
175 'ie_key': ie}
176 return video_info
177 def playlist_result(self, entries, playlist_id=None, playlist_title=None):
178 """Returns a playlist"""
179 video_info = {'_type': 'playlist',
180 'entries': entries}
181 if playlist_id:
182 video_info['id'] = playlist_id
183 if playlist_title:
184 video_info['title'] = playlist_title
185 return video_info
186
187 def _search_regex(self, pattern, string, name, default=None, fatal=True, flags=0):
188 """
189 Perform a regex search on the given string, using a single or a list of
190 patterns returning the first matching group.
191 In case of failure return a default value or raise a WARNING or a
192 ExtractorError, depending on fatal, specifying the field name.
193 """
194 if isinstance(pattern, (str, compat_str, compiled_regex_type)):
195 mobj = re.search(pattern, string, flags)
196 else:
197 for p in pattern:
198 mobj = re.search(p, string, flags)
199 if mobj: break
200
201 if sys.stderr.isatty() and os.name != 'nt':
202 _name = u'\033[0;34m%s\033[0m' % name
203 else:
204 _name = name
205
206 if mobj:
207 # return the first matching group
208 return next(g for g in mobj.groups() if g is not None)
209 elif default is not None:
210 return default
211 elif fatal:
212 raise ExtractorError(u'Unable to extract %s' % _name)
213 else:
214 self._downloader.report_warning(u'unable to extract %s; '
215 u'please report this issue on http://yt-dl.org/bug' % _name)
216 return None
217
218 def _html_search_regex(self, pattern, string, name, default=None, fatal=True, flags=0):
219 """
220 Like _search_regex, but strips HTML tags and unescapes entities.
221 """
222 res = self._search_regex(pattern, string, name, default, fatal, flags)
223 if res:
224 return clean_html(res).strip()
225 else:
226 return res
227
228 class SearchInfoExtractor(InfoExtractor):
229 """
230 Base class for paged search queries extractors.
231 They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
232 Instances should define _SEARCH_KEY and _MAX_RESULTS.
233 """
234
235 @classmethod
236 def _make_valid_url(cls):
237 return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
238
239 @classmethod
240 def suitable(cls, url):
241 return re.match(cls._make_valid_url(), url) is not None
242
243 def _real_extract(self, query):
244 mobj = re.match(self._make_valid_url(), query)
245 if mobj is None:
246 raise ExtractorError(u'Invalid search query "%s"' % query)
247
248 prefix = mobj.group('prefix')
249 query = mobj.group('query')
250 if prefix == '':
251 return self._get_n_results(query, 1)
252 elif prefix == 'all':
253 return self._get_n_results(query, self._MAX_RESULTS)
254 else:
255 n = int(prefix)
256 if n <= 0:
257 raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query))
258 elif n > self._MAX_RESULTS:
259 self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
260 n = self._MAX_RESULTS
261 return self._get_n_results(query, n)
262
263 def _get_n_results(self, query, n):
264 """Get a specified number of results for a query"""
265 raise NotImplementedError("This method must be implemented by sublclasses")
266
267 @property
268 def SEARCH_KEY(self):
269 return self._SEARCH_KEY