]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/viidea.py
d/rules: Run offline tests.
[youtubedl] / youtube_dl / extractor / viidea.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from ..compat import (
7 compat_HTTPError,
8 compat_str,
9 compat_urlparse,
10 )
11 from ..utils import (
12 ExtractorError,
13 js_to_json,
14 parse_duration,
15 parse_iso8601,
16 )
17
18
19 class ViideaIE(InfoExtractor):
20 _VALID_URL = r'''(?x)https?://(?:www\.)?(?:
21 videolectures\.net|
22 flexilearn\.viidea\.net|
23 presentations\.ocwconsortium\.org|
24 video\.travel-zoom\.si|
25 video\.pomp-forum\.si|
26 tv\.nil\.si|
27 video\.hekovnik.com|
28 video\.szko\.si|
29 kpk\.viidea\.com|
30 inside\.viidea\.net|
31 video\.kiberpipa\.org|
32 bvvideo\.si|
33 kongres\.viidea\.net|
34 edemokracija\.viidea\.com
35 )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?/*(?:[#?].*)?$'''
36
37 _TESTS = [{
38 'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
39 'info_dict': {
40 'id': '20171',
41 'display_id': 'promogram_igor_mekjavic_eng',
42 'ext': 'mp4',
43 'title': 'Automatics, robotics and biocybernetics',
44 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
45 'thumbnail': r're:http://.*\.jpg',
46 'timestamp': 1372349289,
47 'upload_date': '20130627',
48 'duration': 565,
49 },
50 'params': {
51 # m3u8 download
52 'skip_download': True,
53 },
54 }, {
55 # video with invalid direct format links (HTTP 403)
56 'url': 'http://videolectures.net/russir2010_filippova_nlp/',
57 'info_dict': {
58 'id': '14891',
59 'display_id': 'russir2010_filippova_nlp',
60 'ext': 'flv',
61 'title': 'NLP at Google',
62 'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
63 'thumbnail': r're:http://.*\.jpg',
64 'timestamp': 1284375600,
65 'upload_date': '20100913',
66 'duration': 5352,
67 },
68 'params': {
69 # rtmp download
70 'skip_download': True,
71 },
72 }, {
73 # event playlist
74 'url': 'http://videolectures.net/deeplearning2015_montreal/',
75 'info_dict': {
76 'id': '23181',
77 'title': 'Deep Learning Summer School, Montreal 2015',
78 'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
79 'thumbnail': r're:http://.*\.jpg',
80 'timestamp': 1438560000,
81 },
82 'playlist_count': 30,
83 }, {
84 # multi part lecture
85 'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
86 'info_dict': {
87 'id': '9737',
88 'display_id': 'mlss09uk_bishop_ibi',
89 'title': 'Introduction To Bayesian Inference',
90 'thumbnail': r're:http://.*\.jpg',
91 'timestamp': 1251622800,
92 },
93 'playlist': [{
94 'info_dict': {
95 'id': '9737_part1',
96 'display_id': 'mlss09uk_bishop_ibi_part1',
97 'ext': 'wmv',
98 'title': 'Introduction To Bayesian Inference (Part 1)',
99 'thumbnail': r're:http://.*\.jpg',
100 'duration': 4622,
101 'timestamp': 1251622800,
102 'upload_date': '20090830',
103 },
104 }, {
105 'info_dict': {
106 'id': '9737_part2',
107 'display_id': 'mlss09uk_bishop_ibi_part2',
108 'ext': 'wmv',
109 'title': 'Introduction To Bayesian Inference (Part 2)',
110 'thumbnail': r're:http://.*\.jpg',
111 'duration': 5641,
112 'timestamp': 1251622800,
113 'upload_date': '20090830',
114 },
115 }],
116 'playlist_count': 2,
117 }]
118
119 def _real_extract(self, url):
120 lecture_slug, explicit_part_id = re.match(self._VALID_URL, url).groups()
121
122 webpage = self._download_webpage(url, lecture_slug)
123
124 cfg = self._parse_json(self._search_regex(
125 [r'cfg\s*:\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*:\s*\(?\s*function',
126 r'cfg\s*:\s*({[^}]+})'],
127 webpage, 'cfg'), lecture_slug, js_to_json)
128
129 lecture_id = compat_str(cfg['obj_id'])
130
131 base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
132
133 try:
134 lecture_data = self._download_json(
135 '%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
136 lecture_id)['lecture'][0]
137 except ExtractorError as e:
138 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
139 msg = self._parse_json(
140 e.cause.read().decode('utf-8'), lecture_id)
141 raise ExtractorError(msg['detail'], expected=True)
142 raise
143
144 lecture_info = {
145 'id': lecture_id,
146 'display_id': lecture_slug,
147 'title': lecture_data['title'],
148 'timestamp': parse_iso8601(lecture_data.get('time')),
149 'description': lecture_data.get('description_wiki'),
150 'thumbnail': lecture_data.get('thumb'),
151 }
152
153 playlist_entries = []
154 lecture_type = lecture_data.get('type')
155 parts = [compat_str(video) for video in cfg.get('videos', [])]
156 if parts:
157 multipart = len(parts) > 1
158
159 def extract_part(part_id):
160 smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part_id)
161 smil = self._download_smil(smil_url, lecture_id)
162 info = self._parse_smil(smil, smil_url, lecture_id)
163 self._sort_formats(info['formats'])
164 info['id'] = lecture_id if not multipart else '%s_part%s' % (lecture_id, part_id)
165 info['display_id'] = lecture_slug if not multipart else '%s_part%s' % (lecture_slug, part_id)
166 if multipart:
167 info['title'] += ' (Part %s)' % part_id
168 switch = smil.find('.//switch')
169 if switch is not None:
170 info['duration'] = parse_duration(switch.attrib.get('dur'))
171 item_info = lecture_info.copy()
172 item_info.update(info)
173 return item_info
174
175 if explicit_part_id or not multipart:
176 result = extract_part(explicit_part_id or parts[0])
177 else:
178 result = {
179 '_type': 'multi_video',
180 'entries': [extract_part(part) for part in parts],
181 }
182 result.update(lecture_info)
183
184 # Immediately return explicitly requested part or non event item
185 if explicit_part_id or lecture_type != 'evt':
186 return result
187
188 playlist_entries.append(result)
189
190 # It's probably a playlist
191 if not parts or lecture_type == 'evt':
192 playlist_webpage = self._download_webpage(
193 '%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
194 entries = [
195 self.url_result(compat_urlparse.urljoin(url, video_url), 'Viidea')
196 for _, video_url in re.findall(
197 r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
198 playlist_entries.extend(entries)
199
200 playlist = self.playlist_result(playlist_entries, lecture_id)
201 playlist.update(lecture_info)
202 return playlist