]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/vier.py
3e67eb8c2ccda6964147919d1420fd4de7198d91
[youtubedl] / youtube_dl / extractor / vier.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5 import itertools
6
7 from .common import InfoExtractor
8 from ..utils import (
9 urlencode_postdata,
10 int_or_none,
11 unified_strdate,
12 )
13
14
15 class VierIE(InfoExtractor):
16 IE_NAME = 'vier'
17 IE_DESC = 'vier.be and vijf.be'
18 _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
19 _NETRC_MACHINE = 'vier'
20 _TESTS = [{
21 'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
22 'md5': 'e4ae2054a6b040ef1e289e20d111b46e',
23 'info_dict': {
24 'id': '16129',
25 'display_id': 'het-wordt-warm-de-moestuin',
26 'ext': 'mp4',
27 'title': 'Het wordt warm in De Moestuin',
28 'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
29 'upload_date': '20121025',
30 'series': 'Plan B',
31 'tags': ['De Moestuin', 'Moestuin', 'meisjes', 'Tomaat', 'Wim', 'Droom'],
32 },
33 }, {
34 'url': 'http://www.vijf.be/temptationisland/videos/zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas/2561614',
35 'info_dict': {
36 'id': '2561614',
37 'display_id': 'zo-grappig-temptation-island-hosts-moeten-kiezen-tussen-onmogelijke-dilemmas',
38 'ext': 'mp4',
39 'title': 'md5:84f45fe48b8c1fa296a7f6d208d080a7',
40 'description': 'md5:0356d4981e58b8cbee19355cbd51a8fe',
41 'upload_date': '20170228',
42 'series': 'Temptation Island',
43 'tags': list,
44 },
45 'params': {
46 'skip_download': True,
47 },
48 }, {
49 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839',
50 'info_dict': {
51 'id': '2674839',
52 'display_id': 'jani-gaat-naar-tokio-aflevering-4',
53 'ext': 'mp4',
54 'title': 'Jani gaat naar Tokio - Aflevering 4',
55 'description': 'md5:aa8d611541db6ae9e863125704511f88',
56 'upload_date': '20170501',
57 'series': 'Jani gaat',
58 'episode_number': 4,
59 'tags': ['Jani Gaat', 'Volledige Aflevering'],
60 },
61 'params': {
62 'skip_download': True,
63 },
64 'skip': 'Requires account credentials',
65 }, {
66 # Requires account credentials but bypassed extraction via v3/embed page
67 # without metadata
68 'url': 'http://www.vier.be/janigaat/videos/jani-gaat-naar-tokio-aflevering-4/2674839',
69 'info_dict': {
70 'id': '2674839',
71 'display_id': 'jani-gaat-naar-tokio-aflevering-4',
72 'ext': 'mp4',
73 'title': 'jani-gaat-naar-tokio-aflevering-4',
74 },
75 'params': {
76 'skip_download': True,
77 },
78 'expected_warnings': ['Log in to extract metadata'],
79 }, {
80 # Without video id in URL
81 'url': 'http://www.vier.be/planb/videos/dit-najaar-plan-b',
82 'only_matching': True,
83 }, {
84 'url': 'http://www.vier.be/video/v3/embed/16129',
85 'only_matching': True,
86 }]
87
88 def _real_initialize(self):
89 self._logged_in = False
90
91 def _login(self, site):
92 username, password = self._get_login_info()
93 if username is None or password is None:
94 return
95
96 login_page = self._download_webpage(
97 'http://www.%s.be/user/login' % site,
98 None, note='Logging in', errnote='Unable to log in',
99 data=urlencode_postdata({
100 'form_id': 'user_login',
101 'name': username,
102 'pass': password,
103 }),
104 headers={'Content-Type': 'application/x-www-form-urlencoded'})
105
106 login_error = self._html_search_regex(
107 r'(?s)<div class="messages error">\s*<div>\s*<h2.+?</h2>(.+?)<',
108 login_page, 'login error', default=None)
109 if login_error:
110 self.report_warning('Unable to log in: %s' % login_error)
111 else:
112 self._logged_in = True
113
114 def _real_extract(self, url):
115 mobj = re.match(self._VALID_URL, url)
116 embed_id = mobj.group('embed_id')
117 display_id = mobj.group('display_id') or embed_id
118 video_id = mobj.group('id') or embed_id
119 site = mobj.group('site')
120
121 if not self._logged_in:
122 self._login(site)
123
124 webpage = self._download_webpage(url, display_id)
125
126 if r'id="user-login"' in webpage:
127 self.report_warning(
128 'Log in to extract metadata', video_id=display_id)
129 webpage = self._download_webpage(
130 'http://www.%s.be/video/v3/embed/%s' % (site, video_id),
131 display_id)
132
133 video_id = self._search_regex(
134 [r'data-nid="(\d+)"', r'"nid"\s*:\s*"(\d+)"'],
135 webpage, 'video id', default=video_id or display_id)
136 application = self._search_regex(
137 [r'data-application="([^"]+)"', r'"application"\s*:\s*"([^"]+)"'],
138 webpage, 'application', default=site + '_vod')
139 filename = self._search_regex(
140 [r'data-filename="([^"]+)"', r'"filename"\s*:\s*"([^"]+)"'],
141 webpage, 'filename')
142
143 playlist_url = 'http://vod.streamcloud.be/%s/_definst_/mp4:%s.mp4/playlist.m3u8' % (application, filename)
144 formats = self._extract_wowza_formats(
145 playlist_url, display_id, skip_protocols=['dash'])
146 self._sort_formats(formats)
147
148 title = self._og_search_title(webpage, default=display_id)
149 description = self._html_search_regex(
150 r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-type-text-with-summary\b[^>]*?\1[^>]*>.*?<p>(?P<value>.+?)</p>',
151 webpage, 'description', default=None, group='value')
152 thumbnail = self._og_search_thumbnail(webpage, default=None)
153 upload_date = unified_strdate(self._html_search_regex(
154 r'(?s)<div\b[^>]+\bclass=(["\'])[^>]*?\bfield-name-post-date\b[^>]*?\1[^>]*>.*?(?P<value>\d{2}/\d{2}/\d{4})',
155 webpage, 'upload date', default=None, group='value'))
156
157 series = self._search_regex(
158 r'data-program=(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
159 'series', default=None, group='value')
160 episode_number = int_or_none(self._search_regex(
161 r'(?i)aflevering (\d+)', title, 'episode number', default=None))
162 tags = re.findall(r'<a\b[^>]+\bhref=["\']/tags/[^>]+>([^<]+)<', webpage)
163
164 return {
165 'id': video_id,
166 'display_id': display_id,
167 'title': title,
168 'description': description,
169 'thumbnail': thumbnail,
170 'upload_date': upload_date,
171 'series': series,
172 'episode_number': episode_number,
173 'tags': tags,
174 'formats': formats,
175 }
176
177
178 class VierVideosIE(InfoExtractor):
179 IE_NAME = 'vier:videos'
180 _VALID_URL = r'https?://(?:www\.)?(?P<site>vier|vijf)\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+)|$)'
181 _TESTS = [{
182 'url': 'http://www.vier.be/demoestuin/videos',
183 'info_dict': {
184 'id': 'demoestuin',
185 },
186 'playlist_mincount': 153,
187 }, {
188 'url': 'http://www.vijf.be/temptationisland/videos',
189 'info_dict': {
190 'id': 'temptationisland',
191 },
192 'playlist_mincount': 159,
193 }, {
194 'url': 'http://www.vier.be/demoestuin/videos?page=6',
195 'info_dict': {
196 'id': 'demoestuin-page6',
197 },
198 'playlist_mincount': 20,
199 }, {
200 'url': 'http://www.vier.be/demoestuin/videos?page=7',
201 'info_dict': {
202 'id': 'demoestuin-page7',
203 },
204 'playlist_mincount': 13,
205 }]
206
207 def _real_extract(self, url):
208 mobj = re.match(self._VALID_URL, url)
209 program = mobj.group('program')
210 site = mobj.group('site')
211
212 page_id = mobj.group('page')
213 if page_id:
214 page_id = int(page_id)
215 start_page = page_id
216 playlist_id = '%s-page%d' % (program, page_id)
217 else:
218 start_page = 0
219 playlist_id = program
220
221 entries = []
222 for current_page_id in itertools.count(start_page):
223 current_page = self._download_webpage(
224 'http://www.%s.be/%s/videos?page=%d' % (site, program, current_page_id),
225 program,
226 'Downloading page %d' % (current_page_id + 1))
227 page_entries = [
228 self.url_result('http://www.' + site + '.be' + video_url, 'Vier')
229 for video_url in re.findall(
230 r'<h[23]><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)]
231 entries.extend(page_entries)
232 if page_id or '>Meer<' not in current_page:
233 break
234
235 return self.playlist_result(entries, playlist_id)