]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/common.py
14f57563529ad5782b3c70c24d4760c78c3b334d
[youtubedl] / youtube_dl / extractor / common.py
1 from __future__ import unicode_literals
2
3 import base64
4 import datetime
5 import hashlib
6 import json
7 import netrc
8 import os
9 import re
10 import socket
11 import sys
12 import time
13 import math
14
15 from ..compat import (
16 compat_cookiejar,
17 compat_cookies,
18 compat_getpass,
19 compat_http_client,
20 compat_urllib_error,
21 compat_urllib_parse,
22 compat_urlparse,
23 compat_str,
24 compat_etree_fromstring,
25 )
26 from ..utils import (
27 NO_DEFAULT,
28 age_restricted,
29 bug_reports_message,
30 clean_html,
31 compiled_regex_type,
32 determine_ext,
33 error_to_compat_str,
34 ExtractorError,
35 fix_xml_ampersands,
36 float_or_none,
37 int_or_none,
38 parse_iso8601,
39 RegexNotFoundError,
40 sanitize_filename,
41 sanitized_Request,
42 unescapeHTML,
43 unified_strdate,
44 url_basename,
45 xpath_text,
46 xpath_with_ns,
47 determine_protocol,
48 parse_duration,
49 mimetype2ext,
50 )
51
52
53 class InfoExtractor(object):
54 """Information Extractor class.
55
56 Information extractors are the classes that, given a URL, extract
57 information about the video (or videos) the URL refers to. This
58 information includes the real video URL, the video title, author and
59 others. The information is stored in a dictionary which is then
60 passed to the YoutubeDL. The YoutubeDL processes this
61 information possibly downloading the video to the file system, among
62 other possible outcomes.
63
64 The type field determines the type of the result.
65 By far the most common value (and the default if _type is missing) is
66 "video", which indicates a single video.
67
68 For a video, the dictionaries must include the following fields:
69
70 id: Video identifier.
71 title: Video title, unescaped.
72
73 Additionally, it must contain either a formats entry or a url one:
74
75 formats: A list of dictionaries for each format available, ordered
76 from worst to best quality.
77
78 Potential fields:
79 * url Mandatory. The URL of the video file
80 * ext Will be calculated from URL if missing
81 * format A human-readable description of the format
82 ("mp4 container with h264/opus").
83 Calculated from the format_id, width, height.
84 and format_note fields if missing.
85 * format_id A short description of the format
86 ("mp4_h264_opus" or "19").
87 Technically optional, but strongly recommended.
88 * format_note Additional info about the format
89 ("3D" or "DASH video")
90 * width Width of the video, if known
91 * height Height of the video, if known
92 * resolution Textual description of width and height
93 * tbr Average bitrate of audio and video in KBit/s
94 * abr Average audio bitrate in KBit/s
95 * acodec Name of the audio codec in use
96 * asr Audio sampling rate in Hertz
97 * vbr Average video bitrate in KBit/s
98 * fps Frame rate
99 * vcodec Name of the video codec in use
100 * container Name of the container format
101 * filesize The number of bytes, if known in advance
102 * filesize_approx An estimate for the number of bytes
103 * player_url SWF Player URL (used for rtmpdump).
104 * protocol The protocol that will be used for the actual
105 download, lower-case.
106 "http", "https", "rtsp", "rtmp", "rtmpe",
107 "m3u8", or "m3u8_native".
108 * preference Order number of this format. If this field is
109 present and not None, the formats get sorted
110 by this field, regardless of all other values.
111 -1 for default (order by other properties),
112 -2 or smaller for less than default.
113 < -1000 to hide the format (if there is
114 another one which is strictly better)
115 * language Language code, e.g. "de" or "en-US".
116 * language_preference Is this in the language mentioned in
117 the URL?
118 10 if it's what the URL is about,
119 -1 for default (don't know),
120 -10 otherwise, other values reserved for now.
121 * quality Order number of the video quality of this
122 format, irrespective of the file format.
123 -1 for default (order by other properties),
124 -2 or smaller for less than default.
125 * source_preference Order number for this video source
126 (quality takes higher priority)
127 -1 for default (order by other properties),
128 -2 or smaller for less than default.
129 * http_headers A dictionary of additional HTTP headers
130 to add to the request.
131 * stretched_ratio If given and not 1, indicates that the
132 video's pixels are not square.
133 width : height ratio as float.
134 * no_resume The server does not support resuming the
135 (HTTP or RTMP) download. Boolean.
136
137 url: Final video URL.
138 ext: Video filename extension.
139 format: The video format, defaults to ext (used for --get-format)
140 player_url: SWF Player URL (used for rtmpdump).
141
142 The following fields are optional:
143
144 alt_title: A secondary title of the video.
145 display_id An alternative identifier for the video, not necessarily
146 unique, but available before title. Typically, id is
147 something like "4234987", title "Dancing naked mole rats",
148 and display_id "dancing-naked-mole-rats"
149 thumbnails: A list of dictionaries, with the following entries:
150 * "id" (optional, string) - Thumbnail format ID
151 * "url"
152 * "preference" (optional, int) - quality of the image
153 * "width" (optional, int)
154 * "height" (optional, int)
155 * "resolution" (optional, string "{width}x{height"},
156 deprecated)
157 thumbnail: Full URL to a video thumbnail image.
158 description: Full video description.
159 uploader: Full name of the video uploader.
160 creator: The main artist who created the video.
161 release_date: The date (YYYYMMDD) when the video was released.
162 timestamp: UNIX timestamp of the moment the video became available.
163 upload_date: Video upload date (YYYYMMDD).
164 If not explicitly set, calculated from timestamp.
165 uploader_id: Nickname or id of the video uploader.
166 location: Physical location where the video was filmed.
167 subtitles: The available subtitles as a dictionary in the format
168 {language: subformats}. "subformats" is a list sorted from
169 lower to higher preference, each element is a dictionary
170 with the "ext" entry and one of:
171 * "data": The subtitles file contents
172 * "url": A URL pointing to the subtitles file
173 "ext" will be calculated from URL if missing
174 automatic_captions: Like 'subtitles', used by the YoutubeIE for
175 automatically generated captions
176 duration: Length of the video in seconds, as an integer or float.
177 view_count: How many users have watched the video on the platform.
178 like_count: Number of positive ratings of the video
179 dislike_count: Number of negative ratings of the video
180 repost_count: Number of reposts of the video
181 average_rating: Average rating give by users, the scale used depends on the webpage
182 comment_count: Number of comments on the video
183 comments: A list of comments, each with one or more of the following
184 properties (all but one of text or html optional):
185 * "author" - human-readable name of the comment author
186 * "author_id" - user ID of the comment author
187 * "id" - Comment ID
188 * "html" - Comment as HTML
189 * "text" - Plain text of the comment
190 * "timestamp" - UNIX timestamp of comment
191 * "parent" - ID of the comment this one is replying to.
192 Set to "root" to indicate that this is a
193 comment to the original video.
194 age_limit: Age restriction for the video, as an integer (years)
195 webpage_url: The URL to the video webpage, if given to youtube-dl it
196 should allow to get the same result again. (It will be set
197 by YoutubeDL if it's missing)
198 categories: A list of categories that the video falls in, for example
199 ["Sports", "Berlin"]
200 tags: A list of tags assigned to the video, e.g. ["sweden", "pop music"]
201 is_live: True, False, or None (=unknown). Whether this video is a
202 live stream that goes on instead of a fixed-length video.
203 start_time: Time in seconds where the reproduction should start, as
204 specified in the URL.
205 end_time: Time in seconds where the reproduction should end, as
206 specified in the URL.
207
208 The following fields should only be used when the video belongs to some logical
209 chapter or section:
210
211 chapter: Name or title of the chapter the video belongs to.
212 chapter_number: Number of the chapter the video belongs to, as an integer.
213 chapter_id: Id of the chapter the video belongs to, as a unicode string.
214
215 The following fields should only be used when the video is an episode of some
216 series or programme:
217
218 series: Title of the series or programme the video episode belongs to.
219 season: Title of the season the video episode belongs to.
220 season_number: Number of the season the video episode belongs to, as an integer.
221 season_id: Id of the season the video episode belongs to, as a unicode string.
222 episode: Title of the video episode. Unlike mandatory video title field,
223 this field should denote the exact title of the video episode
224 without any kind of decoration.
225 episode_number: Number of the video episode within a season, as an integer.
226 episode_id: Id of the video episode, as a unicode string.
227
228 Unless mentioned otherwise, the fields should be Unicode strings.
229
230 Unless mentioned otherwise, None is equivalent to absence of information.
231
232
233 _type "playlist" indicates multiple videos.
234 There must be a key "entries", which is a list, an iterable, or a PagedList
235 object, each element of which is a valid dictionary by this specification.
236
237 Additionally, playlists can have "title", "description" and "id" attributes
238 with the same semantics as videos (see above).
239
240
241 _type "multi_video" indicates that there are multiple videos that
242 form a single show, for examples multiple acts of an opera or TV episode.
243 It must have an entries key like a playlist and contain all the keys
244 required for a video at the same time.
245
246
247 _type "url" indicates that the video must be extracted from another
248 location, possibly by a different extractor. Its only required key is:
249 "url" - the next URL to extract.
250 The key "ie_key" can be set to the class name (minus the trailing "IE",
251 e.g. "Youtube") if the extractor class is known in advance.
252 Additionally, the dictionary may have any properties of the resolved entity
253 known in advance, for example "title" if the title of the referred video is
254 known ahead of time.
255
256
257 _type "url_transparent" entities have the same specification as "url", but
258 indicate that the given additional information is more precise than the one
259 associated with the resolved URL.
260 This is useful when a site employs a video service that hosts the video and
261 its technical metadata, but that video service does not embed a useful
262 title, description etc.
263
264
265 Subclasses of this one should re-define the _real_initialize() and
266 _real_extract() methods and define a _VALID_URL regexp.
267 Probably, they should also be added to the list of extractors.
268
269 Finally, the _WORKING attribute should be set to False for broken IEs
270 in order to warn the users and skip the tests.
271 """
272
273 _ready = False
274 _downloader = None
275 _WORKING = True
276
277 def __init__(self, downloader=None):
278 """Constructor. Receives an optional downloader."""
279 self._ready = False
280 self.set_downloader(downloader)
281
282 @classmethod
283 def suitable(cls, url):
284 """Receives a URL and returns True if suitable for this IE."""
285
286 # This does not use has/getattr intentionally - we want to know whether
287 # we have cached the regexp for *this* class, whereas getattr would also
288 # match the superclass
289 if '_VALID_URL_RE' not in cls.__dict__:
290 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
291 return cls._VALID_URL_RE.match(url) is not None
292
293 @classmethod
294 def _match_id(cls, url):
295 if '_VALID_URL_RE' not in cls.__dict__:
296 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
297 m = cls._VALID_URL_RE.match(url)
298 assert m
299 return m.group('id')
300
301 @classmethod
302 def working(cls):
303 """Getter method for _WORKING."""
304 return cls._WORKING
305
306 def initialize(self):
307 """Initializes an instance (authentication, etc)."""
308 if not self._ready:
309 self._real_initialize()
310 self._ready = True
311
312 def extract(self, url):
313 """Extracts URL information and returns it in list of dicts."""
314 try:
315 self.initialize()
316 return self._real_extract(url)
317 except ExtractorError:
318 raise
319 except compat_http_client.IncompleteRead as e:
320 raise ExtractorError('A network error has occurred.', cause=e, expected=True)
321 except (KeyError, StopIteration) as e:
322 raise ExtractorError('An extractor error has occurred.', cause=e)
323
324 def set_downloader(self, downloader):
325 """Sets the downloader for this IE."""
326 self._downloader = downloader
327
328 def _real_initialize(self):
329 """Real initialization process. Redefine in subclasses."""
330 pass
331
332 def _real_extract(self, url):
333 """Real extraction process. Redefine in subclasses."""
334 pass
335
336 @classmethod
337 def ie_key(cls):
338 """A string for getting the InfoExtractor with get_info_extractor"""
339 return compat_str(cls.__name__[:-2])
340
341 @property
342 def IE_NAME(self):
343 return compat_str(type(self).__name__[:-2])
344
345 def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
346 """ Returns the response handle """
347 if note is None:
348 self.report_download_webpage(video_id)
349 elif note is not False:
350 if video_id is None:
351 self.to_screen('%s' % (note,))
352 else:
353 self.to_screen('%s: %s' % (video_id, note))
354 try:
355 return self._downloader.urlopen(url_or_request)
356 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
357 if errnote is False:
358 return False
359 if errnote is None:
360 errnote = 'Unable to download webpage'
361
362 errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
363 if fatal:
364 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
365 else:
366 self._downloader.report_warning(errmsg)
367 return False
368
369 def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):
370 """ Returns a tuple (page content as string, URL handle) """
371 # Strip hashes from the URL (#1038)
372 if isinstance(url_or_request, (compat_str, str)):
373 url_or_request = url_or_request.partition('#')[0]
374
375 urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
376 if urlh is False:
377 assert not fatal
378 return False
379 content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
380 return (content, urlh)
381
382 @staticmethod
383 def _guess_encoding_from_content(content_type, webpage_bytes):
384 m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
385 if m:
386 encoding = m.group(1)
387 else:
388 m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
389 webpage_bytes[:1024])
390 if m:
391 encoding = m.group(1).decode('ascii')
392 elif webpage_bytes.startswith(b'\xff\xfe'):
393 encoding = 'utf-16'
394 else:
395 encoding = 'utf-8'
396
397 return encoding
398
399 def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
400 content_type = urlh.headers.get('Content-Type', '')
401 webpage_bytes = urlh.read()
402 if prefix is not None:
403 webpage_bytes = prefix + webpage_bytes
404 if not encoding:
405 encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
406 if self._downloader.params.get('dump_intermediate_pages', False):
407 try:
408 url = url_or_request.get_full_url()
409 except AttributeError:
410 url = url_or_request
411 self.to_screen('Dumping request to ' + url)
412 dump = base64.b64encode(webpage_bytes).decode('ascii')
413 self._downloader.to_screen(dump)
414 if self._downloader.params.get('write_pages', False):
415 try:
416 url = url_or_request.get_full_url()
417 except AttributeError:
418 url = url_or_request
419 basen = '%s_%s' % (video_id, url)
420 if len(basen) > 240:
421 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
422 basen = basen[:240 - len(h)] + h
423 raw_filename = basen + '.dump'
424 filename = sanitize_filename(raw_filename, restricted=True)
425 self.to_screen('Saving request to ' + filename)
426 # Working around MAX_PATH limitation on Windows (see
427 # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
428 if os.name == 'nt':
429 absfilepath = os.path.abspath(filename)
430 if len(absfilepath) > 259:
431 filename = '\\\\?\\' + absfilepath
432 with open(filename, 'wb') as outf:
433 outf.write(webpage_bytes)
434
435 try:
436 content = webpage_bytes.decode(encoding, 'replace')
437 except LookupError:
438 content = webpage_bytes.decode('utf-8', 'replace')
439
440 if ('<title>Access to this site is blocked</title>' in content and
441 'Websense' in content[:512]):
442 msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
443 blocked_iframe = self._html_search_regex(
444 r'<iframe src="([^"]+)"', content,
445 'Websense information URL', default=None)
446 if blocked_iframe:
447 msg += ' Visit %s for more details' % blocked_iframe
448 raise ExtractorError(msg, expected=True)
449 if '<title>The URL you requested has been blocked</title>' in content[:512]:
450 msg = (
451 'Access to this webpage has been blocked by Indian censorship. '
452 'Use a VPN or proxy server (with --proxy) to route around it.')
453 block_msg = self._html_search_regex(
454 r'</h1><p>(.*?)</p>',
455 content, 'block message', default=None)
456 if block_msg:
457 msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
458 raise ExtractorError(msg, expected=True)
459
460 return content
461
462 def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
463 """ Returns the data of the page as a string """
464 success = False
465 try_count = 0
466 while success is False:
467 try:
468 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding)
469 success = True
470 except compat_http_client.IncompleteRead as e:
471 try_count += 1
472 if try_count >= tries:
473 raise e
474 self._sleep(timeout, video_id)
475 if res is False:
476 return res
477 else:
478 content, _ = res
479 return content
480
481 def _download_xml(self, url_or_request, video_id,
482 note='Downloading XML', errnote='Unable to download XML',
483 transform_source=None, fatal=True, encoding=None):
484 """Return the xml as an xml.etree.ElementTree.Element"""
485 xml_string = self._download_webpage(
486 url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding)
487 if xml_string is False:
488 return xml_string
489 if transform_source:
490 xml_string = transform_source(xml_string)
491 return compat_etree_fromstring(xml_string.encode('utf-8'))
492
493 def _download_json(self, url_or_request, video_id,
494 note='Downloading JSON metadata',
495 errnote='Unable to download JSON metadata',
496 transform_source=None,
497 fatal=True, encoding=None):
498 json_string = self._download_webpage(
499 url_or_request, video_id, note, errnote, fatal=fatal,
500 encoding=encoding)
501 if (not fatal) and json_string is False:
502 return None
503 return self._parse_json(
504 json_string, video_id, transform_source=transform_source, fatal=fatal)
505
506 def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
507 if transform_source:
508 json_string = transform_source(json_string)
509 try:
510 return json.loads(json_string)
511 except ValueError as ve:
512 errmsg = '%s: Failed to parse JSON ' % video_id
513 if fatal:
514 raise ExtractorError(errmsg, cause=ve)
515 else:
516 self.report_warning(errmsg + str(ve))
517
518 def report_warning(self, msg, video_id=None):
519 idstr = '' if video_id is None else '%s: ' % video_id
520 self._downloader.report_warning(
521 '[%s] %s%s' % (self.IE_NAME, idstr, msg))
522
523 def to_screen(self, msg):
524 """Print msg to screen, prefixing it with '[ie_name]'"""
525 self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
526
527 def report_extraction(self, id_or_name):
528 """Report information extraction."""
529 self.to_screen('%s: Extracting information' % id_or_name)
530
531 def report_download_webpage(self, video_id):
532 """Report webpage download."""
533 self.to_screen('%s: Downloading webpage' % video_id)
534
535 def report_age_confirmation(self):
536 """Report attempt to confirm age."""
537 self.to_screen('Confirming age')
538
539 def report_login(self):
540 """Report attempt to log in."""
541 self.to_screen('Logging in')
542
543 @staticmethod
544 def raise_login_required(msg='This video is only available for registered users'):
545 raise ExtractorError(
546 '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
547 expected=True)
548
549 @staticmethod
550 def raise_geo_restricted(msg='This video is not available from your location due to geo restriction'):
551 raise ExtractorError(
552 '%s. You might want to use --proxy to workaround.' % msg,
553 expected=True)
554
555 # Methods for following #608
556 @staticmethod
557 def url_result(url, ie=None, video_id=None, video_title=None):
558 """Returns a URL that points to a page that should be processed"""
559 # TODO: ie should be the class used for getting the info
560 video_info = {'_type': 'url',
561 'url': url,
562 'ie_key': ie}
563 if video_id is not None:
564 video_info['id'] = video_id
565 if video_title is not None:
566 video_info['title'] = video_title
567 return video_info
568
569 @staticmethod
570 def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
571 """Returns a playlist"""
572 video_info = {'_type': 'playlist',
573 'entries': entries}
574 if playlist_id:
575 video_info['id'] = playlist_id
576 if playlist_title:
577 video_info['title'] = playlist_title
578 if playlist_description:
579 video_info['description'] = playlist_description
580 return video_info
581
582 def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
583 """
584 Perform a regex search on the given string, using a single or a list of
585 patterns returning the first matching group.
586 In case of failure return a default value or raise a WARNING or a
587 RegexNotFoundError, depending on fatal, specifying the field name.
588 """
589 if isinstance(pattern, (str, compat_str, compiled_regex_type)):
590 mobj = re.search(pattern, string, flags)
591 else:
592 for p in pattern:
593 mobj = re.search(p, string, flags)
594 if mobj:
595 break
596
597 if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
598 _name = '\033[0;34m%s\033[0m' % name
599 else:
600 _name = name
601
602 if mobj:
603 if group is None:
604 # return the first matching group
605 return next(g for g in mobj.groups() if g is not None)
606 else:
607 return mobj.group(group)
608 elif default is not NO_DEFAULT:
609 return default
610 elif fatal:
611 raise RegexNotFoundError('Unable to extract %s' % _name)
612 else:
613 self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
614 return None
615
616 def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
617 """
618 Like _search_regex, but strips HTML tags and unescapes entities.
619 """
620 res = self._search_regex(pattern, string, name, default, fatal, flags, group)
621 if res:
622 return clean_html(res).strip()
623 else:
624 return res
625
626 def _get_login_info(self):
627 """
628 Get the login info as (username, password)
629 It will look in the netrc file using the _NETRC_MACHINE value
630 If there's no info available, return (None, None)
631 """
632 if self._downloader is None:
633 return (None, None)
634
635 username = None
636 password = None
637 downloader_params = self._downloader.params
638
639 # Attempt to use provided username and password or .netrc data
640 if downloader_params.get('username') is not None:
641 username = downloader_params['username']
642 password = downloader_params['password']
643 elif downloader_params.get('usenetrc', False):
644 try:
645 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
646 if info is not None:
647 username = info[0]
648 password = info[2]
649 else:
650 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
651 except (IOError, netrc.NetrcParseError) as err:
652 self._downloader.report_warning('parsing .netrc: %s' % error_to_compat_str(err))
653
654 return (username, password)
655
656 def _get_tfa_info(self, note='two-factor verification code'):
657 """
658 Get the two-factor authentication info
659 TODO - asking the user will be required for sms/phone verify
660 currently just uses the command line option
661 If there's no info available, return None
662 """
663 if self._downloader is None:
664 return None
665 downloader_params = self._downloader.params
666
667 if downloader_params.get('twofactor') is not None:
668 return downloader_params['twofactor']
669
670 return compat_getpass('Type %s and press [Return]: ' % note)
671
672 # Helper functions for extracting OpenGraph info
673 @staticmethod
674 def _og_regexes(prop):
675 content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
676 property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
677 % {'prop': re.escape(prop)})
678 template = r'<meta[^>]+?%s[^>]+?%s'
679 return [
680 template % (property_re, content_re),
681 template % (content_re, property_re),
682 ]
683
684 @staticmethod
685 def _meta_regex(prop):
686 return r'''(?isx)<meta
687 (?=[^>]+(?:itemprop|name|property|id|http-equiv)=(["\']?)%s\1)
688 [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
689
690 def _og_search_property(self, prop, html, name=None, **kargs):
691 if name is None:
692 name = 'OpenGraph %s' % prop
693 escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
694 if escaped is None:
695 return None
696 return unescapeHTML(escaped)
697
698 def _og_search_thumbnail(self, html, **kargs):
699 return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)
700
701 def _og_search_description(self, html, **kargs):
702 return self._og_search_property('description', html, fatal=False, **kargs)
703
704 def _og_search_title(self, html, **kargs):
705 return self._og_search_property('title', html, **kargs)
706
707 def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
708 regexes = self._og_regexes('video') + self._og_regexes('video:url')
709 if secure:
710 regexes = self._og_regexes('video:secure_url') + regexes
711 return self._html_search_regex(regexes, html, name, **kargs)
712
713 def _og_search_url(self, html, **kargs):
714 return self._og_search_property('url', html, **kargs)
715
716 def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
717 if display_name is None:
718 display_name = name
719 return self._html_search_regex(
720 self._meta_regex(name),
721 html, display_name, fatal=fatal, group='content', **kwargs)
722
723 def _dc_search_uploader(self, html):
724 return self._html_search_meta('dc.creator', html, 'uploader')
725
726 def _rta_search(self, html):
727 # See http://www.rtalabel.org/index.php?content=howtofaq#single
728 if re.search(r'(?ix)<meta\s+name="rating"\s+'
729 r' content="RTA-5042-1996-1400-1577-RTA"',
730 html):
731 return 18
732 return 0
733
734 def _media_rating_search(self, html):
735 # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
736 rating = self._html_search_meta('rating', html)
737
738 if not rating:
739 return None
740
741 RATING_TABLE = {
742 'safe for kids': 0,
743 'general': 8,
744 '14 years': 14,
745 'mature': 17,
746 'restricted': 19,
747 }
748 return RATING_TABLE.get(rating.lower())
749
750 def _family_friendly_search(self, html):
751 # See http://schema.org/VideoObject
752 family_friendly = self._html_search_meta('isFamilyFriendly', html)
753
754 if not family_friendly:
755 return None
756
757 RATING_TABLE = {
758 '1': 0,
759 'true': 0,
760 '0': 18,
761 'false': 18,
762 }
763 return RATING_TABLE.get(family_friendly.lower())
764
765 def _twitter_search_player(self, html):
766 return self._html_search_meta('twitter:player', html,
767 'twitter card player')
768
769 def _search_json_ld(self, html, video_id, **kwargs):
770 json_ld = self._search_regex(
771 r'(?s)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>',
772 html, 'JSON-LD', group='json_ld', **kwargs)
773 if not json_ld:
774 return {}
775 return self._json_ld(json_ld, video_id, fatal=kwargs.get('fatal', True))
776
777 def _json_ld(self, json_ld, video_id, fatal=True):
778 if isinstance(json_ld, compat_str):
779 json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
780 if not json_ld:
781 return {}
782 info = {}
783 if json_ld.get('@context') == 'http://schema.org':
784 item_type = json_ld.get('@type')
785 if item_type == 'TVEpisode':
786 info.update({
787 'episode': unescapeHTML(json_ld.get('name')),
788 'episode_number': int_or_none(json_ld.get('episodeNumber')),
789 'description': unescapeHTML(json_ld.get('description')),
790 })
791 part_of_season = json_ld.get('partOfSeason')
792 if isinstance(part_of_season, dict) and part_of_season.get('@type') == 'TVSeason':
793 info['season_number'] = int_or_none(part_of_season.get('seasonNumber'))
794 part_of_series = json_ld.get('partOfSeries')
795 if isinstance(part_of_series, dict) and part_of_series.get('@type') == 'TVSeries':
796 info['series'] = unescapeHTML(part_of_series.get('name'))
797 elif item_type == 'Article':
798 info.update({
799 'timestamp': parse_iso8601(json_ld.get('datePublished')),
800 'title': unescapeHTML(json_ld.get('headline')),
801 'description': unescapeHTML(json_ld.get('articleBody')),
802 })
803 return dict((k, v) for k, v in info.items() if v is not None)
804
805 @staticmethod
806 def _hidden_inputs(html):
807 html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
808 hidden_inputs = {}
809 for input in re.findall(r'(?i)<input([^>]+)>', html):
810 if not re.search(r'type=(["\'])(?:hidden|submit)\1', input):
811 continue
812 name = re.search(r'name=(["\'])(?P<value>.+?)\1', input)
813 if not name:
814 continue
815 value = re.search(r'value=(["\'])(?P<value>.*?)\1', input)
816 if not value:
817 continue
818 hidden_inputs[name.group('value')] = value.group('value')
819 return hidden_inputs
820
821 def _form_hidden_inputs(self, form_id, html):
822 form = self._search_regex(
823 r'(?is)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
824 html, '%s form' % form_id, group='form')
825 return self._hidden_inputs(form)
826
827 def _sort_formats(self, formats, field_preference=None):
828 if not formats:
829 raise ExtractorError('No video formats found')
830
831 for f in formats:
832 # Automatically determine tbr when missing based on abr and vbr (improves
833 # formats sorting in some cases)
834 if 'tbr' not in f and f.get('abr') is not None and f.get('vbr') is not None:
835 f['tbr'] = f['abr'] + f['vbr']
836
837 def _formats_key(f):
838 # TODO remove the following workaround
839 from ..utils import determine_ext
840 if not f.get('ext') and 'url' in f:
841 f['ext'] = determine_ext(f['url'])
842
843 if isinstance(field_preference, (list, tuple)):
844 return tuple(f.get(field) if f.get(field) is not None else -1 for field in field_preference)
845
846 preference = f.get('preference')
847 if preference is None:
848 preference = 0
849 if f.get('ext') in ['f4f', 'f4m']: # Not yet supported
850 preference -= 0.5
851
852 proto_preference = 0 if determine_protocol(f) in ['http', 'https'] else -0.1
853
854 if f.get('vcodec') == 'none': # audio only
855 if self._downloader.params.get('prefer_free_formats'):
856 ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
857 else:
858 ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
859 ext_preference = 0
860 try:
861 audio_ext_preference = ORDER.index(f['ext'])
862 except ValueError:
863 audio_ext_preference = -1
864 else:
865 if self._downloader.params.get('prefer_free_formats'):
866 ORDER = ['flv', 'mp4', 'webm']
867 else:
868 ORDER = ['webm', 'flv', 'mp4']
869 try:
870 ext_preference = ORDER.index(f['ext'])
871 except ValueError:
872 ext_preference = -1
873 audio_ext_preference = 0
874
875 return (
876 preference,
877 f.get('language_preference') if f.get('language_preference') is not None else -1,
878 f.get('quality') if f.get('quality') is not None else -1,
879 f.get('tbr') if f.get('tbr') is not None else -1,
880 f.get('filesize') if f.get('filesize') is not None else -1,
881 f.get('vbr') if f.get('vbr') is not None else -1,
882 f.get('height') if f.get('height') is not None else -1,
883 f.get('width') if f.get('width') is not None else -1,
884 proto_preference,
885 ext_preference,
886 f.get('abr') if f.get('abr') is not None else -1,
887 audio_ext_preference,
888 f.get('fps') if f.get('fps') is not None else -1,
889 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
890 f.get('source_preference') if f.get('source_preference') is not None else -1,
891 f.get('format_id') if f.get('format_id') is not None else '',
892 )
893 formats.sort(key=_formats_key)
894
895 def _check_formats(self, formats, video_id):
896 if formats:
897 formats[:] = filter(
898 lambda f: self._is_valid_url(
899 f['url'], video_id,
900 item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
901 formats)
902
903 @staticmethod
904 def _remove_duplicate_formats(formats):
905 format_urls = set()
906 unique_formats = []
907 for f in formats:
908 if f['url'] not in format_urls:
909 format_urls.add(f['url'])
910 unique_formats.append(f)
911 formats[:] = unique_formats
912
913 def _is_valid_url(self, url, video_id, item='video'):
914 url = self._proto_relative_url(url, scheme='http:')
915 # For now assume non HTTP(S) URLs always valid
916 if not (url.startswith('http://') or url.startswith('https://')):
917 return True
918 try:
919 self._request_webpage(url, video_id, 'Checking %s URL' % item)
920 return True
921 except ExtractorError as e:
922 if isinstance(e.cause, compat_urllib_error.URLError):
923 self.to_screen(
924 '%s: %s URL is invalid, skipping' % (video_id, item))
925 return False
926 raise
927
928 def http_scheme(self):
929 """ Either "http:" or "https:", depending on the user's preferences """
930 return (
931 'http:'
932 if self._downloader.params.get('prefer_insecure', False)
933 else 'https:')
934
935 def _proto_relative_url(self, url, scheme=None):
936 if url is None:
937 return url
938 if url.startswith('//'):
939 if scheme is None:
940 scheme = self.http_scheme()
941 return scheme + url
942 else:
943 return url
944
945 def _sleep(self, timeout, video_id, msg_template=None):
946 if msg_template is None:
947 msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
948 msg = msg_template % {'video_id': video_id, 'timeout': timeout}
949 self.to_screen(msg)
950 time.sleep(timeout)
951
952 def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
953 transform_source=lambda s: fix_xml_ampersands(s).strip(),
954 fatal=True):
955 manifest = self._download_xml(
956 manifest_url, video_id, 'Downloading f4m manifest',
957 'Unable to download f4m manifest',
958 # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
959 # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
960 transform_source=transform_source,
961 fatal=fatal)
962
963 if manifest is False:
964 return []
965
966 formats = []
967 manifest_version = '1.0'
968 media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
969 if not media_nodes:
970 manifest_version = '2.0'
971 media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
972 base_url = xpath_text(
973 manifest, ['{http://ns.adobe.com/f4m/1.0}baseURL', '{http://ns.adobe.com/f4m/2.0}baseURL'],
974 'base URL', default=None)
975 if base_url:
976 base_url = base_url.strip()
977 for i, media_el in enumerate(media_nodes):
978 if manifest_version == '2.0':
979 media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
980 if not media_url:
981 continue
982 manifest_url = (
983 media_url if media_url.startswith('http://') or media_url.startswith('https://')
984 else ((base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
985 # If media_url is itself a f4m manifest do the recursive extraction
986 # since bitrates in parent manifest (this one) and media_url manifest
987 # may differ leading to inability to resolve the format by requested
988 # bitrate in f4m downloader
989 if determine_ext(manifest_url) == 'f4m':
990 formats.extend(self._extract_f4m_formats(
991 manifest_url, video_id, preference, f4m_id, fatal=fatal))
992 continue
993 tbr = int_or_none(media_el.attrib.get('bitrate'))
994 formats.append({
995 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),
996 'url': manifest_url,
997 'ext': 'flv',
998 'tbr': tbr,
999 'width': int_or_none(media_el.attrib.get('width')),
1000 'height': int_or_none(media_el.attrib.get('height')),
1001 'preference': preference,
1002 })
1003 self._sort_formats(formats)
1004
1005 return formats
1006
1007 def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
1008 entry_protocol='m3u8', preference=None,
1009 m3u8_id=None, note=None, errnote=None,
1010 fatal=True):
1011
1012 formats = [{
1013 'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
1014 'url': m3u8_url,
1015 'ext': ext,
1016 'protocol': 'm3u8',
1017 'preference': preference - 1 if preference else -1,
1018 'resolution': 'multiple',
1019 'format_note': 'Quality selection URL',
1020 }]
1021
1022 format_url = lambda u: (
1023 u
1024 if re.match(r'^https?://', u)
1025 else compat_urlparse.urljoin(m3u8_url, u))
1026
1027 res = self._download_webpage_handle(
1028 m3u8_url, video_id,
1029 note=note or 'Downloading m3u8 information',
1030 errnote=errnote or 'Failed to download m3u8 information',
1031 fatal=fatal)
1032 if res is False:
1033 return []
1034 m3u8_doc, urlh = res
1035 m3u8_url = urlh.geturl()
1036 # A Media Playlist Tag MUST NOT appear in a Master Playlist
1037 # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3
1038 # The EXT-X-TARGETDURATION tag is REQUIRED for every M3U8 Media Playlists
1039 # https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.1
1040 if '#EXT-X-TARGETDURATION' in m3u8_doc:
1041 return [{
1042 'url': m3u8_url,
1043 'format_id': m3u8_id,
1044 'ext': ext,
1045 'protocol': entry_protocol,
1046 'preference': preference,
1047 }]
1048 last_info = None
1049 last_media = None
1050 kv_rex = re.compile(
1051 r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
1052 for line in m3u8_doc.splitlines():
1053 if line.startswith('#EXT-X-STREAM-INF:'):
1054 last_info = {}
1055 for m in kv_rex.finditer(line):
1056 v = m.group('val')
1057 if v.startswith('"'):
1058 v = v[1:-1]
1059 last_info[m.group('key')] = v
1060 elif line.startswith('#EXT-X-MEDIA:'):
1061 last_media = {}
1062 for m in kv_rex.finditer(line):
1063 v = m.group('val')
1064 if v.startswith('"'):
1065 v = v[1:-1]
1066 last_media[m.group('key')] = v
1067 elif line.startswith('#') or not line.strip():
1068 continue
1069 else:
1070 if last_info is None:
1071 formats.append({'url': format_url(line)})
1072 continue
1073 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
1074 format_id = []
1075 if m3u8_id:
1076 format_id.append(m3u8_id)
1077 last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
1078 format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
1079 f = {
1080 'format_id': '-'.join(format_id),
1081 'url': format_url(line.strip()),
1082 'tbr': tbr,
1083 'ext': ext,
1084 'protocol': entry_protocol,
1085 'preference': preference,
1086 }
1087 codecs = last_info.get('CODECS')
1088 if codecs:
1089 # TODO: looks like video codec is not always necessarily goes first
1090 va_codecs = codecs.split(',')
1091 if va_codecs[0]:
1092 f['vcodec'] = va_codecs[0]
1093 if len(va_codecs) > 1 and va_codecs[1]:
1094 f['acodec'] = va_codecs[1]
1095 resolution = last_info.get('RESOLUTION')
1096 if resolution:
1097 width_str, height_str = resolution.split('x')
1098 f['width'] = int(width_str)
1099 f['height'] = int(height_str)
1100 if last_media is not None:
1101 f['m3u8_media'] = last_media
1102 last_media = None
1103 formats.append(f)
1104 last_info = {}
1105 self._sort_formats(formats)
1106 return formats
1107
1108 @staticmethod
1109 def _xpath_ns(path, namespace=None):
1110 if not namespace:
1111 return path
1112 out = []
1113 for c in path.split('/'):
1114 if not c or c == '.':
1115 out.append(c)
1116 else:
1117 out.append('{%s}%s' % (namespace, c))
1118 return '/'.join(out)
1119
1120 def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None):
1121 smil = self._download_smil(smil_url, video_id, fatal=fatal)
1122
1123 if smil is False:
1124 assert not fatal
1125 return []
1126
1127 namespace = self._parse_smil_namespace(smil)
1128
1129 return self._parse_smil_formats(
1130 smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1131
1132 def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
1133 smil = self._download_smil(smil_url, video_id, fatal=fatal)
1134 if smil is False:
1135 return {}
1136 return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
1137
1138 def _download_smil(self, smil_url, video_id, fatal=True):
1139 return self._download_xml(
1140 smil_url, video_id, 'Downloading SMIL file',
1141 'Unable to download SMIL file', fatal=fatal)
1142
1143 def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
1144 namespace = self._parse_smil_namespace(smil)
1145
1146 formats = self._parse_smil_formats(
1147 smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1148 subtitles = self._parse_smil_subtitles(smil, namespace=namespace)
1149
1150 video_id = os.path.splitext(url_basename(smil_url))[0]
1151 title = None
1152 description = None
1153 upload_date = None
1154 for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1155 name = meta.attrib.get('name')
1156 content = meta.attrib.get('content')
1157 if not name or not content:
1158 continue
1159 if not title and name == 'title':
1160 title = content
1161 elif not description and name in ('description', 'abstract'):
1162 description = content
1163 elif not upload_date and name == 'date':
1164 upload_date = unified_strdate(content)
1165
1166 thumbnails = [{
1167 'id': image.get('type'),
1168 'url': image.get('src'),
1169 'width': int_or_none(image.get('width')),
1170 'height': int_or_none(image.get('height')),
1171 } for image in smil.findall(self._xpath_ns('.//image', namespace)) if image.get('src')]
1172
1173 return {
1174 'id': video_id,
1175 'title': title or video_id,
1176 'description': description,
1177 'upload_date': upload_date,
1178 'thumbnails': thumbnails,
1179 'formats': formats,
1180 'subtitles': subtitles,
1181 }
1182
1183 def _parse_smil_namespace(self, smil):
1184 return self._search_regex(
1185 r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
1186
1187 def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
1188 base = smil_url
1189 for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1190 b = meta.get('base') or meta.get('httpBase')
1191 if b:
1192 base = b
1193 break
1194
1195 formats = []
1196 rtmp_count = 0
1197 http_count = 0
1198 m3u8_count = 0
1199
1200 srcs = []
1201 videos = smil.findall(self._xpath_ns('.//video', namespace))
1202 for video in videos:
1203 src = video.get('src')
1204 if not src or src in srcs:
1205 continue
1206 srcs.append(src)
1207
1208 bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
1209 filesize = int_or_none(video.get('size') or video.get('fileSize'))
1210 width = int_or_none(video.get('width'))
1211 height = int_or_none(video.get('height'))
1212 proto = video.get('proto')
1213 ext = video.get('ext')
1214 src_ext = determine_ext(src)
1215 streamer = video.get('streamer') or base
1216
1217 if proto == 'rtmp' or streamer.startswith('rtmp'):
1218 rtmp_count += 1
1219 formats.append({
1220 'url': streamer,
1221 'play_path': src,
1222 'ext': 'flv',
1223 'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
1224 'tbr': bitrate,
1225 'filesize': filesize,
1226 'width': width,
1227 'height': height,
1228 })
1229 if transform_rtmp_url:
1230 streamer, src = transform_rtmp_url(streamer, src)
1231 formats[-1].update({
1232 'url': streamer,
1233 'play_path': src,
1234 })
1235 continue
1236
1237 src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
1238 src_url = src_url.strip()
1239
1240 if proto == 'm3u8' or src_ext == 'm3u8':
1241 m3u8_formats = self._extract_m3u8_formats(
1242 src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
1243 if len(m3u8_formats) == 1:
1244 m3u8_count += 1
1245 m3u8_formats[0].update({
1246 'format_id': 'hls-%d' % (m3u8_count if bitrate is None else bitrate),
1247 'tbr': bitrate,
1248 'width': width,
1249 'height': height,
1250 })
1251 formats.extend(m3u8_formats)
1252 continue
1253
1254 if src_ext == 'f4m':
1255 f4m_url = src_url
1256 if not f4m_params:
1257 f4m_params = {
1258 'hdcore': '3.2.0',
1259 'plugin': 'flowplayer-3.2.0.1',
1260 }
1261 f4m_url += '&' if '?' in f4m_url else '?'
1262 f4m_url += compat_urllib_parse.urlencode(f4m_params)
1263 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
1264 continue
1265
1266 if src_url.startswith('http') and self._is_valid_url(src, video_id):
1267 http_count += 1
1268 formats.append({
1269 'url': src_url,
1270 'ext': ext or src_ext or 'flv',
1271 'format_id': 'http-%d' % (bitrate or http_count),
1272 'tbr': bitrate,
1273 'filesize': filesize,
1274 'width': width,
1275 'height': height,
1276 })
1277 continue
1278
1279 self._sort_formats(formats)
1280
1281 return formats
1282
1283 def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
1284 urls = []
1285 subtitles = {}
1286 for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
1287 src = textstream.get('src')
1288 if not src or src in urls:
1289 continue
1290 urls.append(src)
1291 ext = textstream.get('ext') or determine_ext(src) or mimetype2ext(textstream.get('type'))
1292 lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
1293 subtitles.setdefault(lang, []).append({
1294 'url': src,
1295 'ext': ext,
1296 })
1297 return subtitles
1298
1299 def _extract_xspf_playlist(self, playlist_url, playlist_id, fatal=True):
1300 xspf = self._download_xml(
1301 playlist_url, playlist_id, 'Downloading xpsf playlist',
1302 'Unable to download xspf manifest', fatal=fatal)
1303 if xspf is False:
1304 return []
1305 return self._parse_xspf(xspf, playlist_id)
1306
1307 def _parse_xspf(self, playlist, playlist_id):
1308 NS_MAP = {
1309 'xspf': 'http://xspf.org/ns/0/',
1310 's1': 'http://static.streamone.nl/player/ns/0',
1311 }
1312
1313 entries = []
1314 for track in playlist.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
1315 title = xpath_text(
1316 track, xpath_with_ns('./xspf:title', NS_MAP), 'title', default=playlist_id)
1317 description = xpath_text(
1318 track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
1319 thumbnail = xpath_text(
1320 track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
1321 duration = float_or_none(
1322 xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), 1000)
1323
1324 formats = [{
1325 'url': location.text,
1326 'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
1327 'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
1328 'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
1329 } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
1330 self._sort_formats(formats)
1331
1332 entries.append({
1333 'id': playlist_id,
1334 'title': title,
1335 'description': description,
1336 'thumbnail': thumbnail,
1337 'duration': duration,
1338 'formats': formats,
1339 })
1340 return entries
1341
1342 def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}):
1343 res = self._download_webpage_handle(
1344 mpd_url, video_id,
1345 note=note or 'Downloading MPD manifest',
1346 errnote=errnote or 'Failed to download MPD manifest',
1347 fatal=fatal)
1348 if res is False:
1349 return []
1350 mpd, urlh = res
1351 mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group()
1352
1353 return self._parse_mpd_formats(
1354 compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict)
1355
1356 def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
1357 if mpd_doc.get('type') == 'dynamic':
1358 return []
1359
1360 namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)
1361
1362 def _add_ns(path):
1363 return self._xpath_ns(path, namespace)
1364
1365 def is_drm_protected(element):
1366 return element.find(_add_ns('ContentProtection')) is not None
1367
1368 def extract_multisegment_info(element, ms_parent_info):
1369 ms_info = ms_parent_info.copy()
1370 segment_list = element.find(_add_ns('SegmentList'))
1371 if segment_list is not None:
1372 segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
1373 if segment_urls_e:
1374 ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
1375 initialization = segment_list.find(_add_ns('Initialization'))
1376 if initialization is not None:
1377 ms_info['initialization_url'] = initialization.attrib['sourceURL']
1378 else:
1379 segment_template = element.find(_add_ns('SegmentTemplate'))
1380 if segment_template is not None:
1381 start_number = segment_template.get('startNumber')
1382 if start_number:
1383 ms_info['start_number'] = int(start_number)
1384 segment_timeline = segment_template.find(_add_ns('SegmentTimeline'))
1385 if segment_timeline is not None:
1386 s_e = segment_timeline.findall(_add_ns('S'))
1387 if s_e:
1388 ms_info['total_number'] = 0
1389 for s in s_e:
1390 ms_info['total_number'] += 1 + int(s.get('r', '0'))
1391 else:
1392 timescale = segment_template.get('timescale')
1393 if timescale:
1394 ms_info['timescale'] = int(timescale)
1395 segment_duration = segment_template.get('duration')
1396 if segment_duration:
1397 ms_info['segment_duration'] = int(segment_duration)
1398 media_template = segment_template.get('media')
1399 if media_template:
1400 ms_info['media_template'] = media_template
1401 initialization = segment_template.get('initialization')
1402 if initialization:
1403 ms_info['initialization_url'] = initialization
1404 else:
1405 initialization = segment_template.find(_add_ns('Initialization'))
1406 if initialization is not None:
1407 ms_info['initialization_url'] = initialization.attrib['sourceURL']
1408 return ms_info
1409
1410 mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
1411 formats = []
1412 for period in mpd_doc.findall(_add_ns('Period')):
1413 period_duration = parse_duration(period.get('duration')) or mpd_duration
1414 period_ms_info = extract_multisegment_info(period, {
1415 'start_number': 1,
1416 'timescale': 1,
1417 })
1418 for adaptation_set in period.findall(_add_ns('AdaptationSet')):
1419 if is_drm_protected(adaptation_set):
1420 continue
1421 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
1422 for representation in adaptation_set.findall(_add_ns('Representation')):
1423 if is_drm_protected(representation):
1424 continue
1425 representation_attrib = adaptation_set.attrib.copy()
1426 representation_attrib.update(representation.attrib)
1427 mime_type = representation_attrib.get('mimeType')
1428 content_type = mime_type.split('/')[0] if mime_type else representation_attrib.get('contentType')
1429 if content_type == 'text':
1430 # TODO implement WebVTT downloading
1431 pass
1432 elif content_type == 'video' or content_type == 'audio':
1433 base_url = ''
1434 for element in (representation, adaptation_set, period, mpd_doc):
1435 base_url_e = element.find(_add_ns('BaseURL'))
1436 if base_url_e is not None:
1437 base_url = base_url_e.text + base_url
1438 if re.match(r'^https?://', base_url):
1439 break
1440 if mpd_base_url and not re.match(r'^https?://', base_url):
1441 if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
1442 mpd_base_url += '/'
1443 base_url = mpd_base_url + base_url
1444 representation_id = representation_attrib.get('id')
1445 lang = representation_attrib.get('lang')
1446 url_el = representation.find(_add_ns('BaseURL'))
1447 filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
1448 f = {
1449 'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
1450 'url': base_url,
1451 'width': int_or_none(representation_attrib.get('width')),
1452 'height': int_or_none(representation_attrib.get('height')),
1453 'tbr': int_or_none(representation_attrib.get('bandwidth'), 1000),
1454 'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
1455 'fps': int_or_none(representation_attrib.get('frameRate')),
1456 'vcodec': 'none' if content_type == 'audio' else representation_attrib.get('codecs'),
1457 'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'),
1458 'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
1459 'format_note': 'DASH %s' % content_type,
1460 'filesize': filesize,
1461 }
1462 representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
1463 if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:
1464 if 'total_number' not in representation_ms_info and 'segment_duration':
1465 segment_duration = float(representation_ms_info['segment_duration']) / float(representation_ms_info['timescale'])
1466 representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
1467 media_template = representation_ms_info['media_template']
1468 media_template = media_template.replace('$RepresentationID$', representation_id)
1469 media_template = re.sub(r'\$(Number|Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
1470 media_template.replace('$$', '$')
1471 representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number, 'Bandwidth': representation_attrib.get('bandwidth')} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
1472 if 'segment_urls' in representation_ms_info:
1473 f.update({
1474 'segment_urls': representation_ms_info['segment_urls'],
1475 'protocol': 'http_dash_segments',
1476 })
1477 if 'initialization_url' in representation_ms_info:
1478 initialization_url = representation_ms_info['initialization_url'].replace('$RepresentationID$', representation_id)
1479 f.update({
1480 'initialization_url': initialization_url,
1481 })
1482 if not f.get('url'):
1483 f['url'] = initialization_url
1484 try:
1485 existing_format = next(
1486 fo for fo in formats
1487 if fo['format_id'] == representation_id)
1488 except StopIteration:
1489 full_info = formats_dict.get(representation_id, {}).copy()
1490 full_info.update(f)
1491 formats.append(full_info)
1492 else:
1493 existing_format.update(f)
1494 else:
1495 self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
1496 self._sort_formats(formats)
1497 return formats
1498
1499 def _live_title(self, name):
1500 """ Generate the title for a live video """
1501 now = datetime.datetime.now()
1502 now_str = now.strftime('%Y-%m-%d %H:%M')
1503 return name + ' ' + now_str
1504
1505 def _int(self, v, name, fatal=False, **kwargs):
1506 res = int_or_none(v, **kwargs)
1507 if 'get_attr' in kwargs:
1508 print(getattr(v, kwargs['get_attr']))
1509 if res is None:
1510 msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
1511 if fatal:
1512 raise ExtractorError(msg)
1513 else:
1514 self._downloader.report_warning(msg)
1515 return res
1516
1517 def _float(self, v, name, fatal=False, **kwargs):
1518 res = float_or_none(v, **kwargs)
1519 if res is None:
1520 msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
1521 if fatal:
1522 raise ExtractorError(msg)
1523 else:
1524 self._downloader.report_warning(msg)
1525 return res
1526
1527 def _set_cookie(self, domain, name, value, expire_time=None):
1528 cookie = compat_cookiejar.Cookie(
1529 0, name, value, None, None, domain, None,
1530 None, '/', True, False, expire_time, '', None, None, None)
1531 self._downloader.cookiejar.set_cookie(cookie)
1532
1533 def _get_cookies(self, url):
1534 """ Return a compat_cookies.SimpleCookie with the cookies for the url """
1535 req = sanitized_Request(url)
1536 self._downloader.cookiejar.add_cookie_header(req)
1537 return compat_cookies.SimpleCookie(req.get_header('Cookie'))
1538
1539 def get_testcases(self, include_onlymatching=False):
1540 t = getattr(self, '_TEST', None)
1541 if t:
1542 assert not hasattr(self, '_TESTS'), \
1543 '%s has _TEST and _TESTS' % type(self).__name__
1544 tests = [t]
1545 else:
1546 tests = getattr(self, '_TESTS', [])
1547 for t in tests:
1548 if not include_onlymatching and t.get('only_matching', False):
1549 continue
1550 t['name'] = type(self).__name__[:-len('IE')]
1551 yield t
1552
1553 def is_suitable(self, age_limit):
1554 """ Test whether the extractor is generally suitable for the given
1555 age limit (i.e. pornographic sites are not, all others usually are) """
1556
1557 any_restricted = False
1558 for tc in self.get_testcases(include_onlymatching=False):
1559 if 'playlist' in tc:
1560 tc = tc['playlist'][0]
1561 is_restricted = age_restricted(
1562 tc.get('info_dict', {}).get('age_limit'), age_limit)
1563 if not is_restricted:
1564 return True
1565 any_restricted = any_restricted or is_restricted
1566 return not any_restricted
1567
1568 def extract_subtitles(self, *args, **kwargs):
1569 if (self._downloader.params.get('writesubtitles', False) or
1570 self._downloader.params.get('listsubtitles')):
1571 return self._get_subtitles(*args, **kwargs)
1572 return {}
1573
1574 def _get_subtitles(self, *args, **kwargs):
1575 raise NotImplementedError('This method must be implemented by subclasses')
1576
1577 @staticmethod
1578 def _merge_subtitle_items(subtitle_list1, subtitle_list2):
1579 """ Merge subtitle items for one language. Items with duplicated URLs
1580 will be dropped. """
1581 list1_urls = set([item['url'] for item in subtitle_list1])
1582 ret = list(subtitle_list1)
1583 ret.extend([item for item in subtitle_list2 if item['url'] not in list1_urls])
1584 return ret
1585
1586 @classmethod
1587 def _merge_subtitles(cls, subtitle_dict1, subtitle_dict2):
1588 """ Merge two subtitle dictionaries, language by language. """
1589 ret = dict(subtitle_dict1)
1590 for lang in subtitle_dict2:
1591 ret[lang] = cls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
1592 return ret
1593
1594 def extract_automatic_captions(self, *args, **kwargs):
1595 if (self._downloader.params.get('writeautomaticsub', False) or
1596 self._downloader.params.get('listsubtitles')):
1597 return self._get_automatic_captions(*args, **kwargs)
1598 return {}
1599
1600 def _get_automatic_captions(self, *args, **kwargs):
1601 raise NotImplementedError('This method must be implemented by subclasses')
1602
1603
1604 class SearchInfoExtractor(InfoExtractor):
1605 """
1606 Base class for paged search queries extractors.
1607 They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
1608 Instances should define _SEARCH_KEY and _MAX_RESULTS.
1609 """
1610
1611 @classmethod
1612 def _make_valid_url(cls):
1613 return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
1614
1615 @classmethod
1616 def suitable(cls, url):
1617 return re.match(cls._make_valid_url(), url) is not None
1618
1619 def _real_extract(self, query):
1620 mobj = re.match(self._make_valid_url(), query)
1621 if mobj is None:
1622 raise ExtractorError('Invalid search query "%s"' % query)
1623
1624 prefix = mobj.group('prefix')
1625 query = mobj.group('query')
1626 if prefix == '':
1627 return self._get_n_results(query, 1)
1628 elif prefix == 'all':
1629 return self._get_n_results(query, self._MAX_RESULTS)
1630 else:
1631 n = int(prefix)
1632 if n <= 0:
1633 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
1634 elif n > self._MAX_RESULTS:
1635 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
1636 n = self._MAX_RESULTS
1637 return self._get_n_results(query, n)
1638
1639 def _get_n_results(self, query, n):
1640 """Get a specified number of results for a query"""
1641 raise NotImplementedError('This method must be implemented by subclasses')
1642
1643 @property
1644 def SEARCH_KEY(self):
1645 return self._SEARCH_KEY