]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/animeondemand.py
Update the changelog.
[youtubedl] / youtube_dl / extractor / animeondemand.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from ..compat import (
7 compat_urlparse,
8 compat_str,
9 )
10 from ..utils import (
11 determine_ext,
12 extract_attributes,
13 ExtractorError,
14 sanitized_Request,
15 urlencode_postdata,
16 )
17
18
19 class AnimeOnDemandIE(InfoExtractor):
20 _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
21 _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
22 _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
23 _NETRC_MACHINE = 'animeondemand'
24 _TESTS = [{
25 'url': 'https://www.anime-on-demand.de/anime/161',
26 'info_dict': {
27 'id': '161',
28 'title': 'Grimgar, Ashes and Illusions (OmU)',
29 'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
30 },
31 'playlist_mincount': 4,
32 }, {
33 # Film wording is used instead of Episode
34 'url': 'https://www.anime-on-demand.de/anime/39',
35 'only_matching': True,
36 }, {
37 # Episodes without titles
38 'url': 'https://www.anime-on-demand.de/anime/162',
39 'only_matching': True,
40 }, {
41 # ger/jap, Dub/OmU, account required
42 'url': 'https://www.anime-on-demand.de/anime/169',
43 'only_matching': True,
44 }]
45
46 def _login(self):
47 (username, password) = self._get_login_info()
48 if username is None:
49 return
50
51 login_page = self._download_webpage(
52 self._LOGIN_URL, None, 'Downloading login page')
53
54 if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
55 self.raise_geo_restricted(
56 '%s is only available in German-speaking countries of Europe' % self.IE_NAME)
57
58 login_form = self._form_hidden_inputs('new_user', login_page)
59
60 login_form.update({
61 'user[login]': username,
62 'user[password]': password,
63 })
64
65 post_url = self._search_regex(
66 r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
67 'post url', default=self._LOGIN_URL, group='url')
68
69 if not post_url.startswith('http'):
70 post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
71
72 request = sanitized_Request(
73 post_url, urlencode_postdata(login_form))
74 request.add_header('Referer', self._LOGIN_URL)
75
76 response = self._download_webpage(
77 request, None, 'Logging in as %s' % username)
78
79 if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
80 error = self._search_regex(
81 r'<p class="alert alert-danger">(.+?)</p>',
82 response, 'error', default=None)
83 if error:
84 raise ExtractorError('Unable to login: %s' % error, expected=True)
85 raise ExtractorError('Unable to log in')
86
87 def _real_initialize(self):
88 self._login()
89
90 def _real_extract(self, url):
91 anime_id = self._match_id(url)
92
93 webpage = self._download_webpage(url, anime_id)
94
95 if 'data-playlist=' not in webpage:
96 self._download_webpage(
97 self._APPLY_HTML5_URL, anime_id,
98 'Activating HTML5 beta', 'Unable to apply HTML5 beta')
99 webpage = self._download_webpage(url, anime_id)
100
101 csrf_token = self._html_search_meta(
102 'csrf-token', webpage, 'csrf token', fatal=True)
103
104 anime_title = self._html_search_regex(
105 r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
106 webpage, 'anime name')
107 anime_description = self._html_search_regex(
108 r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
109 webpage, 'anime description', default=None)
110
111 entries = []
112
113 for num, episode_html in enumerate(re.findall(
114 r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', webpage), 1):
115 episodebox_title = self._search_regex(
116 (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
117 r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
118 episode_html, 'episodebox title', default=None, group='title')
119 if not episodebox_title:
120 continue
121
122 episode_number = int(self._search_regex(
123 r'(?:Episode|Film)\s*(\d+)',
124 episodebox_title, 'episode number', default=num))
125 episode_title = self._search_regex(
126 r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
127 episodebox_title, 'episode title', default=None)
128
129 video_id = 'episode-%d' % episode_number
130
131 common_info = {
132 'id': video_id,
133 'series': anime_title,
134 'episode': episode_title,
135 'episode_number': episode_number,
136 }
137
138 formats = []
139
140 for input_ in re.findall(
141 r'<input[^>]+class=["\'].*?streamstarter_html5[^>]+>', episode_html):
142 attributes = extract_attributes(input_)
143 playlist_urls = []
144 for playlist_key in ('data-playlist', 'data-otherplaylist'):
145 playlist_url = attributes.get(playlist_key)
146 if isinstance(playlist_url, compat_str) and re.match(
147 r'/?[\da-zA-Z]+', playlist_url):
148 playlist_urls.append(attributes[playlist_key])
149 if not playlist_urls:
150 continue
151
152 lang = attributes.get('data-lang')
153 lang_note = attributes.get('value')
154
155 for playlist_url in playlist_urls:
156 kind = self._search_regex(
157 r'videomaterialurl/\d+/([^/]+)/',
158 playlist_url, 'media kind', default=None)
159 format_id_list = []
160 if lang:
161 format_id_list.append(lang)
162 if kind:
163 format_id_list.append(kind)
164 if not format_id_list:
165 format_id_list.append(compat_str(num))
166 format_id = '-'.join(format_id_list)
167 format_note = ', '.join(filter(None, (kind, lang_note)))
168 request = sanitized_Request(
169 compat_urlparse.urljoin(url, playlist_url),
170 headers={
171 'X-Requested-With': 'XMLHttpRequest',
172 'X-CSRF-Token': csrf_token,
173 'Referer': url,
174 'Accept': 'application/json, text/javascript, */*; q=0.01',
175 })
176 playlist = self._download_json(
177 request, video_id, 'Downloading %s playlist JSON' % format_id,
178 fatal=False)
179 if not playlist:
180 continue
181 start_video = playlist.get('startvideo', 0)
182 playlist = playlist.get('playlist')
183 if not playlist or not isinstance(playlist, list):
184 continue
185 playlist = playlist[start_video]
186 title = playlist.get('title')
187 if not title:
188 continue
189 description = playlist.get('description')
190 for source in playlist.get('sources', []):
191 file_ = source.get('file')
192 if not file_:
193 continue
194 ext = determine_ext(file_)
195 format_id_list = [lang, kind]
196 if ext == 'm3u8':
197 format_id_list.append('hls')
198 elif source.get('type') == 'video/dash' or ext == 'mpd':
199 format_id_list.append('dash')
200 format_id = '-'.join(filter(None, format_id_list))
201 if ext == 'm3u8':
202 file_formats = self._extract_m3u8_formats(
203 file_, video_id, 'mp4',
204 entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
205 elif source.get('type') == 'video/dash' or ext == 'mpd':
206 continue
207 file_formats = self._extract_mpd_formats(
208 file_, video_id, mpd_id=format_id, fatal=False)
209 else:
210 continue
211 for f in file_formats:
212 f.update({
213 'language': lang,
214 'format_note': format_note,
215 })
216 formats.extend(file_formats)
217
218 if formats:
219 self._sort_formats(formats)
220 f = common_info.copy()
221 f.update({
222 'title': title,
223 'description': description,
224 'formats': formats,
225 })
226 entries.append(f)
227
228 # Extract teaser only when full episode is not available
229 if not formats:
230 m = re.search(
231 r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>Teaser<',
232 episode_html)
233 if m:
234 f = common_info.copy()
235 f.update({
236 'id': '%s-teaser' % f['id'],
237 'title': m.group('title'),
238 'url': compat_urlparse.urljoin(url, m.group('href')),
239 })
240 entries.append(f)
241
242 return self.playlist_result(entries, anime_id, anime_title, anime_description)