]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/generic.py
Rapsys Git / youtubedl / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
debian/watch: Skip pgpsigurlmangle for the moment.
[youtubedl] / youtube_dl / extractor / generic.py
1 # encoding: utf-8
2
3 from __future__ import unicode_literals
4
5 import os
6 import re
7 import sys
8
9 from .common import InfoExtractor
10 from .youtube import YoutubeIE
11 from ..compat import (
12 compat_etree_fromstring,
13 compat_urllib_parse_unquote,
14 compat_urlparse,
15 compat_xml_parse_error,
16 )
17 from ..utils import (
18 determine_ext,
19 ExtractorError,
20 float_or_none,
21 HEADRequest,
22 is_html,
23 orderedSet,
24 sanitized_Request,
25 smuggle_url,
26 unescapeHTML,
27 unified_strdate,
28 unsmuggle_url,
29 UnsupportedError,
30 url_basename,
31 xpath_text,
32 )
33 from .brightcove import (
34 BrightcoveLegacyIE,
35 BrightcoveNewIE,
36 )
37 from .nbc import NBCSportsVPlayerIE
38 from .ooyala import OoyalaIE
39 from .rutv import RUTVIE
40 from .tvc import TVCIE
41 from .sportbox import SportBoxEmbedIE
42 from .smotri import SmotriIE
43 from .myvi import MyviIE
44 from .condenast import CondeNastIE
45 from .udn import UDNEmbedIE
46 from .senateisvp import SenateISVPIE
47 from .svt import SVTIE
48 from .pornhub import PornHubIE
49 from .xhamster import XHamsterEmbedIE
50 from .vimeo import VimeoIE
51 from .dailymotion import DailymotionCloudIE
52 from .onionstudios import OnionStudiosIE
53 from .snagfilms import SnagFilmsEmbedIE
54 from .screenwavemedia import ScreenwaveMediaIE
55 from .mtv import MTVServicesEmbeddedIE
56 from .pladform import PladformIE
57 from .videomore import VideomoreIE
58 from .googledrive import GoogleDriveIE
59 from .jwplatform import JWPlatformIE
60 from .digiteka import DigitekaIE
61
62
63 class GenericIE(InfoExtractor):
64 IE_DESC = 'Generic downloader that works on some sites'
65 _VALID_URL = r'.*'
66 IE_NAME = 'generic'
67 _TESTS = [
68 # Direct link to a video
69 {
70 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
71 'md5': '67d406c2bcb6af27fa886f31aa934bbe',
72 'info_dict': {
73 'id': 'trailer',
74 'ext': 'mp4',
75 'title': 'trailer',
76 'upload_date': '20100513',
77 }
78 },
79 # Direct link to media delivered compressed (until Accept-Encoding is *)
80 {
81 'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
82 'md5': '128c42e68b13950268b648275386fc74',
83 'info_dict': {
84 'id': 'FictionJunction-Parallel_Hearts',
85 'ext': 'flac',
86 'title': 'FictionJunction-Parallel_Hearts',
87 'upload_date': '20140522',
88 },
89 'expected_warnings': [
90 'URL could be a direct video link, returning it as such.'
91 ]
92 },
93 # Direct download with broken HEAD
94 {
95 'url': 'http://ai-radio.org:8000/radio.opus',
96 'info_dict': {
97 'id': 'radio',
98 'ext': 'opus',
99 'title': 'radio',
100 },
101 'params': {
102 'skip_download': True, # infinite live stream
103 },
104 'expected_warnings': [
105 r'501.*Not Implemented'
106 ],
107 },
108 # Direct link with incorrect MIME type
109 {
110 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
111 'md5': '4ccbebe5f36706d85221f204d7eb5913',
112 'info_dict': {
113 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
114 'id': '5_Lennart_Poettering_-_Systemd',
115 'ext': 'webm',
116 'title': '5_Lennart_Poettering_-_Systemd',
117 'upload_date': '20141120',
118 },
119 'expected_warnings': [
120 'URL could be a direct video link, returning it as such.'
121 ]
122 },
123 # RSS feed
124 {
125 'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
126 'info_dict': {
127 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
128 'title': 'Zero Punctuation',
129 'description': 're:.*groundbreaking video review series.*'
130 },
131 'playlist_mincount': 11,
132 },
133 # RSS feed with enclosure
134 {
135 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
136 'info_dict': {
137 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
138 'ext': 'm4v',
139 'upload_date': '20150228',
140 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
141 }
142 },
143 # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
144 {
145 'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
146 'info_dict': {
147 'id': 'smil',
148 'ext': 'mp4',
149 'title': 'Automatics, robotics and biocybernetics',
150 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
151 'upload_date': '20130627',
152 'formats': 'mincount:16',
153 'subtitles': 'mincount:1',
154 },
155 'params': {
156 'force_generic_extractor': True,
157 'skip_download': True,
158 },
159 },
160 # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
161 {
162 'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
163 'info_dict': {
164 'id': 'hds',
165 'ext': 'flv',
166 'title': 'hds',
167 'formats': 'mincount:1',
168 },
169 'params': {
170 'skip_download': True,
171 },
172 },
173 # SMIL from https://www.restudy.dk/video/play/id/1637
174 {
175 'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
176 'info_dict': {
177 'id': 'video_1637',
178 'ext': 'flv',
179 'title': 'video_1637',
180 'formats': 'mincount:3',
181 },
182 'params': {
183 'skip_download': True,
184 },
185 },
186 # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
187 {
188 'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
189 'info_dict': {
190 'id': 'smil-service',
191 'ext': 'flv',
192 'title': 'smil-service',
193 'formats': 'mincount:1',
194 },
195 'params': {
196 'skip_download': True,
197 },
198 },
199 # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
200 {
201 'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
202 'info_dict': {
203 'id': '4719370',
204 'ext': 'mp4',
205 'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
206 'formats': 'mincount:3',
207 },
208 'params': {
209 'skip_download': True,
210 },
211 },
212 # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
213 {
214 'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
215 'info_dict': {
216 'id': 'mZlp2ctYIUEB',
217 'ext': 'mp4',
218 'title': 'Tikibad ontruimd wegens brand',
219 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
220 'thumbnail': 're:^https?://.*\.jpg$',
221 'duration': 33,
222 },
223 'params': {
224 'skip_download': True,
225 },
226 },
227 # MPD from http://dash-mse-test.appspot.com/media.html
228 {
229 'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
230 'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
231 'info_dict': {
232 'id': 'car-20120827-manifest',
233 'ext': 'mp4',
234 'title': 'car-20120827-manifest',
235 'formats': 'mincount:9',
236 },
237 'params': {
238 'format': 'bestvideo',
239 },
240 },
241 # google redirect
242 {
243 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
244 'info_dict': {
245 'id': 'cmQHVoWB5FY',
246 'ext': 'mp4',
247 'upload_date': '20130224',
248 'uploader_id': 'TheVerge',
249 'description': 're:^Chris Ziegler takes a look at the\.*',
250 'uploader': 'The Verge',
251 'title': 'First Firefox OS phones side-by-side',
252 },
253 'params': {
254 'skip_download': False,
255 }
256 },
257 {
258 # redirect in Refresh HTTP header
259 'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
260 'info_dict': {
261 'id': 'pO8h3EaFRdo',
262 'ext': 'mp4',
263 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
264 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
265 'upload_date': '20150917',
266 'uploader_id': 'brtvofficial',
267 'uploader': 'Boiler Room',
268 },
269 'params': {
270 'skip_download': False,
271 },
272 },
273 {
274 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
275 'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
276 'info_dict': {
277 'id': '13601338388002',
278 'ext': 'mp4',
279 'uploader': 'www.hodiho.fr',
280 'title': 'R\u00e9gis plante sa Jeep',
281 }
282 },
283 # bandcamp page with custom domain
284 {
285 'add_ie': ['Bandcamp'],
286 'url': 'http://bronyrock.com/track/the-pony-mash',
287 'info_dict': {
288 'id': '3235767654',
289 'ext': 'mp3',
290 'title': 'The Pony Mash',
291 'uploader': 'M_Pallante',
292 },
293 'skip': 'There is a limit of 200 free downloads / month for the test song',
294 },
295 # embedded brightcove video
296 # it also tests brightcove videos that need to set the 'Referer' in the
297 # http requests
298 {
299 'add_ie': ['BrightcoveLegacy'],
300 'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
301 'info_dict': {
302 'id': '2765128793001',
303 'ext': 'mp4',
304 'title': 'Le cours de bourse : lā€™analyse technique',
305 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
306 'uploader': 'BFM BUSINESS',
307 },
308 'params': {
309 'skip_download': True,
310 },
311 },
312 {
313 # https://github.com/rg3/youtube-dl/issues/2253
314 'url': 'http://bcove.me/i6nfkrc3',
315 'md5': '0ba9446db037002366bab3b3eb30c88c',
316 'info_dict': {
317 'id': '3101154703001',
318 'ext': 'mp4',
319 'title': 'Still no power',
320 'uploader': 'thestar.com',
321 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
322 },
323 'add_ie': ['BrightcoveLegacy'],
324 },
325 {
326 'url': 'http://www.championat.com/video/football/v/87/87499.html',
327 'md5': 'fb973ecf6e4a78a67453647444222983',
328 'info_dict': {
329 'id': '3414141473001',
330 'ext': 'mp4',
331 'title': 'Š’ŠøŠ“ŠµŠ¾. Š£Š“Š°Š»ŠµŠ½ŠøŠµ Š”Š·Š°Š³Š¾ŠµŠ²Š° (Š¦Š”ŠšŠ)',
332 'description': 'ŠžŠ½Š»Š°Š¹Š½-трŠ°Š½ŃŠ»ŃŃ†Šøя Š¼Š°Ń‚чŠ° Š¦Š”ŠšŠ - "Š’Š¾Š»Š³Š°"',
333 'uploader': 'Championat',
334 },
335 },
336 {
337 # https://github.com/rg3/youtube-dl/issues/3541
338 'add_ie': ['BrightcoveLegacy'],
339 'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
340 'info_dict': {
341 'id': '3866516442001',
342 'ext': 'mp4',
343 'title': 'Leer mij vrouwen kennen: Aflevering 1',
344 'description': 'Leer mij vrouwen kennen: Aflevering 1',
345 'uploader': 'SBS Broadcasting',
346 },
347 'skip': 'Restricted to Netherlands',
348 'params': {
349 'skip_download': True, # m3u8 download
350 },
351 },
352 # ooyala video
353 {
354 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
355 'md5': '166dd577b433b4d4ebfee10b0824d8ff',
356 'info_dict': {
357 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
358 'ext': 'mp4',
359 'title': '2cc213299525360.mov', # that's what we get
360 'duration': 238.231,
361 },
362 'add_ie': ['Ooyala'],
363 },
364 {
365 # ooyala video embedded with http://player.ooyala.com/iframe.js
366 'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
367 'info_dict': {
368 'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
369 'ext': 'mp4',
370 'title': '"Steve Jobs: Man in the Machine" trailer',
371 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
372 'duration': 135.427,
373 },
374 'params': {
375 'skip_download': True,
376 },
377 },
378 # multiple ooyala embeds on SBN network websites
379 {
380 'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
381 'info_dict': {
382 'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
383 'title': '25 lies you will tell yourself on National Signing Day - SBNation.com',
384 },
385 'playlist_mincount': 3,
386 'params': {
387 'skip_download': True,
388 },
389 'add_ie': ['Ooyala'],
390 },
391 # embed.ly video
392 {
393 'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
394 'info_dict': {
395 'id': '9ODmcdjQcHQ',
396 'ext': 'mp4',
397 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
398 'upload_date': '20140225',
399 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
400 'uploader': 'Tested',
401 'uploader_id': 'testedcom',
402 },
403 # No need to test YoutubeIE here
404 'params': {
405 'skip_download': True,
406 },
407 },
408 # funnyordie embed
409 {
410 'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
411 'info_dict': {
412 'id': '18e820ec3f',
413 'ext': 'mp4',
414 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
415 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
416 },
417 },
418 # RUTV embed
419 {
420 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
421 'info_dict': {
422 'id': '776940',
423 'ext': 'mp4',
424 'title': 'ŠžŃ…Š¾Ń‚сŠŗŠ¾Šµ Š¼Š¾Ń€Šµ стŠ°Š»Š¾ цŠµŠ»ŠøŠŗŠ¾Š¼ рŠ¾ŃŃŠøŠ¹ŃŠŗŠøŠ¼',
425 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
426 },
427 'params': {
428 # m3u8 download
429 'skip_download': True,
430 },
431 },
432 # TVC embed
433 {
434 'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
435 'info_dict': {
436 'id': '55304',
437 'ext': 'mp4',
438 'title': 'Š”Š¾ŃˆŠŗŠ¾Š»ŃŒŠ½Š¾Šµ Š²Š¾ŃŠæŠøтŠ°Š½ŠøŠµ',
439 },
440 },
441 # SportBox embed
442 {
443 'url': 'http://www.vestifinance.ru/articles/25753',
444 'info_dict': {
445 'id': '25753',
446 'title': 'Š’ŠµŃŃ‚Šø Š­ŠŗŠ¾Š½Š¾Š¼ŠøŠŗŠ° ā€• ŠŸŃ€ŃŠ¼Ń‹Šµ трŠ°Š½ŃŠ»ŃŃ†ŠøŠø с Š¤Š¾Ń€ŃƒŠ¼Š°-Š²Ń‹ŃŃ‚Š°Š²ŠŗŠø "Š“Š¾ŃŠ·Š°ŠŗŠ°Š·-2013"',
447 },
448 'playlist': [{
449 'info_dict': {
450 'id': '370908',
451 'title': 'Š“Š¾ŃŠ·Š°ŠŗŠ°Š·. Š”ŠµŠ½ŃŒ 3',
452 'ext': 'mp4',
453 }
454 }, {
455 'info_dict': {
456 'id': '370905',
457 'title': 'Š“Š¾ŃŠ·Š°ŠŗŠ°Š·. Š”ŠµŠ½ŃŒ 2',
458 'ext': 'mp4',
459 }
460 }, {
461 'info_dict': {
462 'id': '370902',
463 'title': 'Š“Š¾ŃŠ·Š°ŠŗŠ°Š·. Š”ŠµŠ½ŃŒ 1',
464 'ext': 'mp4',
465 }
466 }],
467 'params': {
468 # m3u8 download
469 'skip_download': True,
470 },
471 },
472 # Myvi.ru embed
473 {
474 'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
475 'info_dict': {
476 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
477 'ext': 'mp4',
478 'title': 'Š£Š¶Š°ŃŃ‚ŠøŠŗŠø, руссŠŗŠøŠ¹ трŠµŠ¹Š»ŠµŃ€ (2015)',
479 'thumbnail': 're:^https?://.*\.jpg$',
480 'duration': 153,
481 }
482 },
483 # XHamster embed
484 {
485 'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
486 'info_dict': {
487 'id': 'showthread',
488 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
489 },
490 'playlist_mincount': 7,
491 },
492 # Embedded TED video
493 {
494 'url': 'http://en.support.wordpress.com/videos/ted-talks/',
495 'md5': '65fdff94098e4a607385a60c5177c638',
496 'info_dict': {
497 'id': '1969',
498 'ext': 'mp4',
499 'title': 'Hidden miracles of the natural world',
500 'uploader': 'Louie Schwartzberg',
501 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
502 }
503 },
504 # Embedded Ustream video
505 {
506 'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
507 'md5': '27b99cdb639c9b12a79bca876a073417',
508 'info_dict': {
509 'id': '45734260',
510 'ext': 'flv',
511 'uploader': 'AU SPA: The NSA and Privacy',
512 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
513 }
514 },
515 # nowvideo embed hidden behind percent encoding
516 {
517 'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
518 'md5': '2baf4ddd70f697d94b1c18cf796d5107',
519 'info_dict': {
520 'id': '06e53103ca9aa',
521 'ext': 'flv',
522 'title': 'Macross Episode 001 Watch Macross Episode 001 onl',
523 'description': 'No description',
524 },
525 },
526 # arte embed
527 {
528 'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
529 'md5': '7653032cbb25bf6c80d80f217055fa43',
530 'info_dict': {
531 'id': '048195-004_PLUS7-F',
532 'ext': 'flv',
533 'title': 'X:enius',
534 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
535 'upload_date': '20140320',
536 },
537 'params': {
538 'skip_download': 'Requires rtmpdump'
539 }
540 },
541 # francetv embed
542 {
543 'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
544 'info_dict': {
545 'id': 'EV_30231',
546 'ext': 'mp4',
547 'title': 'Alcaline, le concert avec Calogero',
548 'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
549 'upload_date': '20150226',
550 'timestamp': 1424989860,
551 'duration': 5400,
552 },
553 'params': {
554 # m3u8 downloads
555 'skip_download': True,
556 },
557 'expected_warnings': [
558 'Forbidden'
559 ]
560 },
561 # CondƩ Nast embed
562 {
563 'url': 'http://www.wired.com/2014/04/honda-asimo/',
564 'md5': 'ba0dfe966fa007657bd1443ee672db0f',
565 'info_dict': {
566 'id': '53501be369702d3275860000',
567 'ext': 'mp4',
568 'title': 'Hondaā€™s New Asimo Robot Is More Human Than Ever',
569 }
570 },
571 # Dailymotion embed
572 {
573 'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
574 'md5': '441aeeb82eb72c422c7f14ec533999cd',
575 'info_dict': {
576 'id': 'k2mm4bCdJ6CQ2i7c8o2',
577 'ext': 'mp4',
578 'title': 'Le Zap de Spi0n nĀ°216 - Zapping du Web',
579 'uploader': 'Spi0n',
580 },
581 'add_ie': ['Dailymotion'],
582 },
583 # YouTube embed
584 {
585 'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
586 'info_dict': {
587 'id': 'FXRb4ykk4S0',
588 'ext': 'mp4',
589 'title': 'The NBL Auction 2014',
590 'uploader': 'BADMINTON England',
591 'uploader_id': 'BADMINTONEvents',
592 'upload_date': '20140603',
593 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
594 },
595 'add_ie': ['Youtube'],
596 'params': {
597 'skip_download': True,
598 }
599 },
600 # MTVSercices embed
601 {
602 'url': 'http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too',
603 'md5': '35727f82f58c76d996fc188f9755b0d5',
604 'info_dict': {
605 'id': '0306a69b-8adf-4fb5-aace-75f8e8cbfca9',
606 'ext': 'mp4',
607 'title': 'Review',
608 'description': 'Mario\'s life in the fast lane has never looked so good.',
609 },
610 },
611 # YouTube embed via <data-embed-url="">
612 {
613 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
614 'info_dict': {
615 'id': '4vAffPZIT44',
616 'ext': 'mp4',
617 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
618 'uploader': 'Gameloft',
619 'uploader_id': 'gameloft',
620 'upload_date': '20140828',
621 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
622 },
623 'params': {
624 'skip_download': True,
625 }
626 },
627 # Camtasia studio
628 {
629 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
630 'playlist': [{
631 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
632 'info_dict': {
633 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
634 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
635 'ext': 'flv',
636 'duration': 2235.90,
637 }
638 }, {
639 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
640 'info_dict': {
641 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
642 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
643 'ext': 'flv',
644 'duration': 2235.93,
645 }
646 }],
647 'info_dict': {
648 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
649 }
650 },
651 # Flowplayer
652 {
653 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
654 'md5': '9d65602bf31c6e20014319c7d07fba27',
655 'info_dict': {
656 'id': '5123ea6d5e5a7',
657 'ext': 'mp4',
658 'age_limit': 18,
659 'uploader': 'www.handjobhub.com',
660 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
661 }
662 },
663 # Multiple brightcove videos
664 # https://github.com/rg3/youtube-dl/issues/2283
665 {
666 'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
667 'info_dict': {
668 'id': 'always-never',
669 'title': 'Always / Never - The New Yorker',
670 },
671 'playlist_count': 3,
672 'params': {
673 'extract_flat': False,
674 'skip_download': True,
675 }
676 },
677 # MLB embed
678 {
679 'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
680 'md5': '96f09a37e44da40dd083e12d9a683327',
681 'info_dict': {
682 'id': '33322633',
683 'ext': 'mp4',
684 'title': 'Ump changes call to ball',
685 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
686 'duration': 48,
687 'timestamp': 1401537900,
688 'upload_date': '20140531',
689 'thumbnail': 're:^https?://.*\.jpg$',
690 },
691 },
692 # Wistia embed
693 {
694 'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
695 'md5': '8788b683c777a5cf25621eaf286d0c23',
696 'info_dict': {
697 'id': '1cfaf6b7ea',
698 'ext': 'mov',
699 'title': 'md5:51364a8d3d009997ba99656004b5e20d',
700 'duration': 643.0,
701 'filesize': 182808282,
702 'uploader': 'education-portal.com',
703 },
704 },
705 {
706 'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
707 'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
708 'info_dict': {
709 'id': 'uxjb0lwrcz',
710 'ext': 'mp4',
711 'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
712 'duration': 1715.0,
713 'uploader': 'thoughtworks.wistia.com',
714 },
715 },
716 # Soundcloud embed
717 {
718 'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
719 'info_dict': {
720 'id': '174391317',
721 'ext': 'mp3',
722 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
723 'uploader': 'Sophos Security',
724 'title': 'Chet Chat 171 - Oct 29, 2014',
725 'upload_date': '20141029',
726 }
727 },
728 # Livestream embed
729 {
730 'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
731 'info_dict': {
732 'id': '67864563',
733 'ext': 'flv',
734 'upload_date': '20141112',
735 'title': 'Rosetta #CometLanding webcast HL 10',
736 }
737 },
738 # LazyYT
739 {
740 'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
741 'info_dict': {
742 'id': '1986',
743 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
744 },
745 'playlist_mincount': 2,
746 },
747 # Cinchcast embed
748 {
749 'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
750 'info_dict': {
751 'id': '7141703',
752 'ext': 'mp3',
753 'upload_date': '20141126',
754 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
755 }
756 },
757 # Cinerama player
758 {
759 'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
760 'info_dict': {
761 'id': '730m_DandD_1901_512k',
762 'ext': 'mp4',
763 'uploader': 'www.abc.net.au',
764 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
765 }
766 },
767 # embedded viddler video
768 {
769 'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
770 'info_dict': {
771 'id': '4d03aad9',
772 'ext': 'mp4',
773 'uploader': 'deadspin',
774 'title': 'WALL-TO-GORTAT',
775 'timestamp': 1422285291,
776 'upload_date': '20150126',
777 },
778 'add_ie': ['Viddler'],
779 },
780 # Libsyn embed
781 {
782 'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
783 'info_dict': {
784 'id': '3377616',
785 'ext': 'mp3',
786 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
787 'description': 'md5:601cb790edd05908957dae8aaa866465',
788 'upload_date': '20150220',
789 },
790 },
791 # jwplayer YouTube
792 {
793 'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
794 'info_dict': {
795 'id': 'Mrj4DVp2zeA',
796 'ext': 'mp4',
797 'upload_date': '20150212',
798 'uploader': 'The National Archives UK',
799 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
800 'uploader_id': 'NationalArchives08',
801 'title': 'Webinar: Using Discovery, The National Archivesā€™ online catalogue',
802 },
803 },
804 # rtl.nl embed
805 {
806 'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
807 'playlist_mincount': 5,
808 'info_dict': {
809 'id': 'aanslagen-kopenhagen',
810 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
811 }
812 },
813 # Zapiks embed
814 {
815 'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
816 'info_dict': {
817 'id': '118046',
818 'ext': 'mp4',
819 'title': 'EP3S5 - Bon AppƩtit - Baqueira Mi Corazon !',
820 }
821 },
822 # Kaltura embed
823 {
824 'url': 'http://www.monumentalnetwork.com/videos/john-carlson-postgame-2-25-15',
825 'info_dict': {
826 'id': '1_eergr3h1',
827 'ext': 'mp4',
828 'upload_date': '20150226',
829 'uploader_id': 'MonumentalSports-Kaltura@perfectsensedigital.com',
830 'timestamp': int,
831 'title': 'John Carlson Postgame 2/25/15',
832 },
833 },
834 # Kaltura embed (different embed code)
835 {
836 'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
837 'info_dict': {
838 'id': '1_a52wc67y',
839 'ext': 'flv',
840 'upload_date': '20150127',
841 'uploader_id': 'PremierMedia',
842 'timestamp': int,
843 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
844 },
845 },
846 # Kaltura embed protected with referrer
847 {
848 'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
849 'info_dict': {
850 'id': '1_g4fbemnq',
851 'ext': 'mp4',
852 'title': 'Violetta - Achter De Schermen - Ruggero',
853 'description': 'Achter de schermen met Ruggero',
854 'timestamp': 1435133761,
855 'upload_date': '20150624',
856 'uploader_id': 'echojecka',
857 },
858 },
859 # Eagle.Platform embed (generic URL)
860 {
861 'url': 'http://lenta.ru/news/2015/03/06/navalny/',
862 'info_dict': {
863 'id': '227304',
864 'ext': 'mp4',
865 'title': 'ŠŠ°Š²Š°Š»ŃŒŠ½Ń‹Š¹ Š²Ń‹ŃˆŠµŠ» Š½Š° сŠ²Š¾Š±Š¾Š“у',
866 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
867 'thumbnail': 're:^https?://.*\.jpg$',
868 'duration': 87,
869 'view_count': int,
870 'age_limit': 0,
871 },
872 },
873 # ClipYou (Eagle.Platform) embed (custom URL)
874 {
875 'url': 'http://muz-tv.ru/play/7129/',
876 'info_dict': {
877 'id': '12820',
878 'ext': 'mp4',
879 'title': "'O Sole Mio",
880 'thumbnail': 're:^https?://.*\.jpg$',
881 'duration': 216,
882 'view_count': int,
883 },
884 },
885 # Pladform embed
886 {
887 'url': 'http://muz-tv.ru/kinozal/view/7400/',
888 'info_dict': {
889 'id': '100183293',
890 'ext': 'mp4',
891 'title': 'Š¢Š°Š¹Š½Ń‹ ŠæŠµŃ€ŠµŠ²Š°Š»Š° Š”ŃŃ‚Š»Š¾Š²Š° ā€¢ 1 сŠµŃ€Šøя 2 чŠ°ŃŃ‚ŃŒ',
892 'description': 'Š”Š¾ŠŗуŠ¼ŠµŠ½Ń‚Š°Š»ŃŒŠ½Ń‹Š¹ сŠµŃ€ŠøŠ°Š»-рŠ°ŃŃŠ»ŠµŠ“Š¾Š²Š°Š½ŠøŠµ Š¾Š“Š½Š¾Š¹ ŠøŠ· сŠ°Š¼Ń‹Ń… Š¶ŃƒŃ‚ŠŗŠøх тŠ°Š¹Š½ Š„Š„ Š²ŠµŠŗŠ°',
893 'thumbnail': 're:^https?://.*\.jpg$',
894 'duration': 694,
895 'age_limit': 0,
896 },
897 },
898 # Playwire embed
899 {
900 'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
901 'info_dict': {
902 'id': '3519514',
903 'ext': 'mp4',
904 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
905 'thumbnail': 're:^https?://.*\.png$',
906 'duration': 45.115,
907 },
908 },
909 # 5min embed
910 {
911 'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
912 'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
913 'info_dict': {
914 'id': '518726732',
915 'ext': 'mp4',
916 'title': 'Facebook Creates "On This Day" | Crunch Report',
917 },
918 },
919 # SVT embed
920 {
921 'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
922 'info_dict': {
923 'id': '2900353',
924 'ext': 'flv',
925 'title': 'HƤr trycker Jagr till Giroux (under SVT-intervjun)',
926 'duration': 27,
927 'age_limit': 0,
928 },
929 },
930 # Crooks and Liars embed
931 {
932 'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
933 'info_dict': {
934 'id': '8RUoRhRi',
935 'ext': 'mp4',
936 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
937 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
938 'timestamp': 1428207000,
939 'upload_date': '20150405',
940 'uploader': 'Heather',
941 },
942 },
943 # Crooks and Liars external embed
944 {
945 'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
946 'info_dict': {
947 'id': 'MTE3MjUtMzQ2MzA',
948 'ext': 'mp4',
949 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
950 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
951 'timestamp': 1265032391,
952 'upload_date': '20100201',
953 'uploader': 'Heather',
954 },
955 },
956 # NBC Sports vplayer embed
957 {
958 'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
959 'info_dict': {
960 'id': 'ln7x1qSThw4k',
961 'ext': 'flv',
962 'title': "PFT Live: New leader in the 'new-look' defense",
963 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
964 },
965 },
966 # UDN embed
967 {
968 'url': 'http://www.udn.com/news/story/7314/822787',
969 'md5': 'fd2060e988c326991037b9aff9df21a6',
970 'info_dict': {
971 'id': '300346',
972 'ext': 'mp4',
973 'title': 'äø­äø€äø­ē”·åø«č®Šę€§ å…Øę ”åø«ē”ŸåŠ›ęŒŗ',
974 'thumbnail': 're:^https?://.*\.jpg$',
975 }
976 },
977 # Ooyala embed
978 {
979 'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
980 'info_dict': {
981 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
982 'ext': 'mp4',
983 'description': 'VIDEO: INDEX/MATCH versus VLOOKUP.',
984 'title': 'This is what separates the Excel masters from the wannabes',
985 'duration': 191.933,
986 },
987 'params': {
988 # m3u8 downloads
989 'skip_download': True,
990 }
991 },
992 # Contains a SMIL manifest
993 {
994 'url': 'http://www.telewebion.com/fa/1263668/%D9%82%D8%B1%D8%B9%D9%87%E2%80%8C%DA%A9%D8%B4%DB%8C-%D9%84%DB%8C%DA%AF-%D9%82%D9%87%D8%B1%D9%85%D8%A7%D9%86%D8%A7%D9%86-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7/%2B-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84.html',
995 'info_dict': {
996 'id': 'file',
997 'ext': 'flv',
998 'title': '+ Football: Lottery Champions League Europe',
999 'uploader': 'www.telewebion.com',
1000 },
1001 'params': {
1002 # rtmpe downloads
1003 'skip_download': True,
1004 }
1005 },
1006 # Brightcove URL in single quotes
1007 {
1008 'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
1009 'md5': '4ae374f1f8b91c889c4b9203c8c752af',
1010 'info_dict': {
1011 'id': '4255764656001',
1012 'ext': 'mp4',
1013 'title': 'SN Presents: Russell Martin, World Citizen',
1014 'description': 'To understand why he was the Toronto Blue Jaysā€™ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
1015 'uploader': 'Rogers Sportsnet',
1016 },
1017 },
1018 # Dailymotion Cloud video
1019 {
1020 'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
1021 'md5': '49444254273501a64675a7e68c502681',
1022 'info_dict': {
1023 'id': '5585de919473990de4bee11b',
1024 'ext': 'mp4',
1025 'title': 'Le dƩbat',
1026 'thumbnail': 're:^https?://.*\.jpe?g$',
1027 }
1028 },
1029 # OnionStudios embed
1030 {
1031 'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
1032 'info_dict': {
1033 'id': '2855',
1034 'ext': 'mp4',
1035 'title': 'Donā€™t Understand Bitcoin? This Man Will Mumble An Explanation At You',
1036 'thumbnail': 're:^https?://.*\.jpe?g$',
1037 'uploader': 'ClickHole',
1038 'uploader_id': 'clickhole',
1039 }
1040 },
1041 # SnagFilms embed
1042 {
1043 'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
1044 'info_dict': {
1045 'id': '74849a00-85a9-11e1-9660-123139220831',
1046 'ext': 'mp4',
1047 'title': '#whilewewatch',
1048 }
1049 },
1050 # AdobeTVVideo embed
1051 {
1052 'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
1053 'md5': '43662b577c018ad707a63766462b1e87',
1054 'info_dict': {
1055 'id': '2456',
1056 'ext': 'mp4',
1057 'title': 'New experience with Acrobat DC',
1058 'description': 'New experience with Acrobat DC',
1059 'duration': 248.667,
1060 },
1061 },
1062 # ScreenwaveMedia embed
1063 {
1064 'url': 'http://www.thecinemasnob.com/the-cinema-snob/a-nightmare-on-elm-street-2-freddys-revenge1',
1065 'md5': '24ace5baba0d35d55c6810b51f34e9e0',
1066 'info_dict': {
1067 'id': 'cinemasnob-55d26273809dd',
1068 'ext': 'mp4',
1069 'title': 'cinemasnob',
1070 },
1071 },
1072 # BrightcoveInPageEmbed embed
1073 {
1074 'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
1075 'info_dict': {
1076 'id': '4238694884001',
1077 'ext': 'flv',
1078 'title': 'Tabletop: Dread, Last Thoughts',
1079 'description': 'Tabletop: Dread, Last Thoughts',
1080 'duration': 51690,
1081 },
1082 },
1083 # JWPlayer with M3U8
1084 {
1085 'url': 'http://ren.tv/novosti/2015-09-25/sluchaynyy-prohozhiy-poymal-avtougonshchika-v-murmanske-video',
1086 'info_dict': {
1087 'id': 'playlist',
1088 'ext': 'mp4',
1089 'title': 'Š”Š»ŃƒŃ‡Š°Š¹Š½Ń‹Š¹ ŠæрŠ¾Ń…Š¾Š¶ŠøŠ¹ ŠæŠ¾Š¹Š¼Š°Š» Š°Š²Ń‚Š¾ŃƒŠ³Š¾Š½Ń‰ŠøŠŗŠ° Š² ŠœŃƒŃ€Š¼Š°Š½ŃŠŗŠµ. Š’Š˜Š”Š•Šž | Š Š•Š Š¢Š’',
1090 'uploader': 'ren.tv',
1091 },
1092 'params': {
1093 # m3u8 downloads
1094 'skip_download': True,
1095 }
1096 }
1097 ]
1098
1099 def report_following_redirect(self, new_url):
1100 """Report information extraction."""
1101 self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
1102
1103 def _extract_rss(self, url, video_id, doc):
1104 playlist_title = doc.find('./channel/title').text
1105 playlist_desc_el = doc.find('./channel/description')
1106 playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
1107
1108 entries = []
1109 for it in doc.findall('./channel/item'):
1110 next_url = xpath_text(it, 'link', fatal=False)
1111 if not next_url:
1112 enclosure_nodes = it.findall('./enclosure')
1113 for e in enclosure_nodes:
1114 next_url = e.attrib.get('url')
1115 if next_url:
1116 break
1117
1118 if not next_url:
1119 continue
1120
1121 entries.append({
1122 '_type': 'url',
1123 'url': next_url,
1124 'title': it.find('title').text,
1125 })
1126
1127 return {
1128 '_type': 'playlist',
1129 'id': url,
1130 'title': playlist_title,
1131 'description': playlist_desc,
1132 'entries': entries,
1133 }
1134
1135 def _extract_camtasia(self, url, video_id, webpage):
1136 """ Returns None if no camtasia video can be found. """
1137
1138 camtasia_cfg = self._search_regex(
1139 r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
1140 webpage, 'camtasia configuration file', default=None)
1141 if camtasia_cfg is None:
1142 return None
1143
1144 title = self._html_search_meta('DC.title', webpage, fatal=True)
1145
1146 camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
1147 camtasia_cfg = self._download_xml(
1148 camtasia_url, video_id,
1149 note='Downloading camtasia configuration',
1150 errnote='Failed to download camtasia configuration')
1151 fileset_node = camtasia_cfg.find('./playlist/array/fileset')
1152
1153 entries = []
1154 for n in fileset_node.getchildren():
1155 url_n = n.find('./uri')
1156 if url_n is None:
1157 continue
1158
1159 entries.append({
1160 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
1161 'title': '%s - %s' % (title, n.tag),
1162 'url': compat_urlparse.urljoin(url, url_n.text),
1163 'duration': float_or_none(n.find('./duration').text),
1164 })
1165
1166 return {
1167 '_type': 'playlist',
1168 'entries': entries,
1169 'title': title,
1170 }
1171
1172 def _real_extract(self, url):
1173 if url.startswith('//'):
1174 return {
1175 '_type': 'url',
1176 'url': self.http_scheme() + url,
1177 }
1178
1179 parsed_url = compat_urlparse.urlparse(url)
1180 if not parsed_url.scheme:
1181 default_search = self._downloader.params.get('default_search')
1182 if default_search is None:
1183 default_search = 'fixup_error'
1184
1185 if default_search in ('auto', 'auto_warning', 'fixup_error'):
1186 if '/' in url:
1187 self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
1188 return self.url_result('http://' + url)
1189 elif default_search != 'fixup_error':
1190 if default_search == 'auto_warning':
1191 if re.match(r'^(?:url|URL)$', url):
1192 raise ExtractorError(
1193 'Invalid URL: %r . Call youtube-dl like this: youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url,
1194 expected=True)
1195 else:
1196 self._downloader.report_warning(
1197 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url)
1198 return self.url_result('ytsearch:' + url)
1199
1200 if default_search in ('error', 'fixup_error'):
1201 raise ExtractorError(
1202 '%r is not a valid URL. '
1203 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
1204 % (url, url), expected=True)
1205 else:
1206 if ':' not in default_search:
1207 default_search += ':'
1208 return self.url_result(default_search + url)
1209
1210 url, smuggled_data = unsmuggle_url(url)
1211 force_videoid = None
1212 is_intentional = smuggled_data and smuggled_data.get('to_generic')
1213 if smuggled_data and 'force_videoid' in smuggled_data:
1214 force_videoid = smuggled_data['force_videoid']
1215 video_id = force_videoid
1216 else:
1217 video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
1218
1219 self.to_screen('%s: Requesting header' % video_id)
1220
1221 head_req = HEADRequest(url)
1222 head_response = self._request_webpage(
1223 head_req, video_id,
1224 note=False, errnote='Could not send HEAD request to %s' % url,
1225 fatal=False)
1226
1227 if head_response is not False:
1228 # Check for redirect
1229 new_url = head_response.geturl()
1230 if url != new_url:
1231 self.report_following_redirect(new_url)
1232 if force_videoid:
1233 new_url = smuggle_url(
1234 new_url, {'force_videoid': force_videoid})
1235 return self.url_result(new_url)
1236
1237 full_response = None
1238 if head_response is False:
1239 request = sanitized_Request(url)
1240 request.add_header('Accept-Encoding', '*')
1241 full_response = self._request_webpage(request, video_id)
1242 head_response = full_response
1243
1244 # Check for direct link to a video
1245 content_type = head_response.headers.get('Content-Type', '')
1246 m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>.+)$', content_type)
1247 if m:
1248 upload_date = unified_strdate(
1249 head_response.headers.get('Last-Modified'))
1250 formats = []
1251 if m.group('format_id').endswith('mpegurl'):
1252 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
1253 else:
1254 formats = [{
1255 'format_id': m.group('format_id'),
1256 'url': url,
1257 'vcodec': 'none' if m.group('type') == 'audio' else None
1258 }]
1259 return {
1260 'id': video_id,
1261 'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
1262 'direct': True,
1263 'formats': formats,
1264 'upload_date': upload_date,
1265 }
1266
1267 if not self._downloader.params.get('test', False) and not is_intentional:
1268 force = self._downloader.params.get('force_generic_extractor', False)
1269 self._downloader.report_warning(
1270 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
1271
1272 if not full_response:
1273 request = sanitized_Request(url)
1274 # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
1275 # making it impossible to download only chunk of the file (yet we need only 512kB to
1276 # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
1277 # that will always result in downloading the whole file that is not desirable.
1278 # Therefore for extraction pass we have to override Accept-Encoding to any in order
1279 # to accept raw bytes and being able to download only a chunk.
1280 # It may probably better to solve this by checking Content-Type for application/octet-stream
1281 # after HEAD request finishes, but not sure if we can rely on this.
1282 request.add_header('Accept-Encoding', '*')
1283 full_response = self._request_webpage(request, video_id)
1284
1285 # Maybe it's a direct link to a video?
1286 # Be careful not to download the whole thing!
1287 first_bytes = full_response.read(512)
1288 if not is_html(first_bytes):
1289 self._downloader.report_warning(
1290 'URL could be a direct video link, returning it as such.')
1291 upload_date = unified_strdate(
1292 head_response.headers.get('Last-Modified'))
1293 return {
1294 'id': video_id,
1295 'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
1296 'direct': True,
1297 'url': url,
1298 'upload_date': upload_date,
1299 }
1300
1301 webpage = self._webpage_read_content(
1302 full_response, url, video_id, prefix=first_bytes)
1303
1304 self.report_extraction(video_id)
1305
1306 # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
1307 try:
1308 doc = compat_etree_fromstring(webpage.encode('utf-8'))
1309 if doc.tag == 'rss':
1310 return self._extract_rss(url, video_id, doc)
1311 elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
1312 return self._parse_smil(doc, url, video_id)
1313 elif doc.tag == '{http://xspf.org/ns/0/}playlist':
1314 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
1315 elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
1316 return {
1317 'id': video_id,
1318 'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
1319 'formats': self._parse_mpd_formats(
1320 doc, video_id, mpd_base_url=url.rpartition('/')[0]),
1321 }
1322 except compat_xml_parse_error:
1323 pass
1324
1325 # Is it a Camtasia project?
1326 camtasia_res = self._extract_camtasia(url, video_id, webpage)
1327 if camtasia_res is not None:
1328 return camtasia_res
1329
1330 # Sometimes embedded video player is hidden behind percent encoding
1331 # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
1332 # Unescaping the whole page allows to handle those cases in a generic way
1333 webpage = compat_urllib_parse_unquote(webpage)
1334
1335 # it's tempting to parse this further, but you would
1336 # have to take into account all the variations like
1337 # Video Title - Site Name
1338 # Site Name | Video Title
1339 # Video Title - Tagline | Site Name
1340 # and so on and so forth; it's just not practical
1341 video_title = self._html_search_regex(
1342 r'(?s)<title>(.*?)</title>', webpage, 'video title',
1343 default='video')
1344
1345 # Try to detect age limit automatically
1346 age_limit = self._rta_search(webpage)
1347 # And then there are the jokers who advertise that they use RTA,
1348 # but actually don't.
1349 AGE_LIMIT_MARKERS = [
1350 r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
1351 ]
1352 if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
1353 age_limit = 18
1354
1355 # video uploader is domain name
1356 video_uploader = self._search_regex(
1357 r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
1358
1359 # Helper method
1360 def _playlist_from_matches(matches, getter=None, ie=None):
1361 urlrs = orderedSet(
1362 self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
1363 for m in matches)
1364 return self.playlist_result(
1365 urlrs, playlist_id=video_id, playlist_title=video_title)
1366
1367 # Look for Brightcove Legacy Studio embeds
1368 bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
1369 if bc_urls:
1370 self.to_screen('Brightcove video detected.')
1371 entries = [{
1372 '_type': 'url',
1373 'url': smuggle_url(bc_url, {'Referer': url}),
1374 'ie_key': 'BrightcoveLegacy'
1375 } for bc_url in bc_urls]
1376
1377 return {
1378 '_type': 'playlist',
1379 'title': video_title,
1380 'id': video_id,
1381 'entries': entries,
1382 }
1383
1384 # Look for Brightcove New Studio embeds
1385 bc_urls = BrightcoveNewIE._extract_urls(webpage)
1386 if bc_urls:
1387 return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
1388
1389 # Look for embedded rtl.nl player
1390 matches = re.findall(
1391 r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
1392 webpage)
1393 if matches:
1394 return _playlist_from_matches(matches, ie='RtlNl')
1395
1396 vimeo_url = VimeoIE._extract_vimeo_url(url, webpage)
1397 if vimeo_url is not None:
1398 return self.url_result(vimeo_url)
1399
1400 vid_me_embed_url = self._search_regex(
1401 r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
1402 webpage, 'vid.me embed', default=None)
1403 if vid_me_embed_url is not None:
1404 return self.url_result(vid_me_embed_url, 'Vidme')
1405
1406 # Look for embedded YouTube player
1407 matches = re.findall(r'''(?x)
1408 (?:
1409 <iframe[^>]+?src=|
1410 data-video-url=|
1411 <embed[^>]+?src=|
1412 embedSWF\(?:\s*|
1413 new\s+SWFObject\(
1414 )
1415 (["\'])
1416 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1417 (?:embed|v|p)/.+?)
1418 \1''', webpage)
1419 if matches:
1420 return _playlist_from_matches(
1421 matches, lambda m: unescapeHTML(m[1]))
1422
1423 # Look for lazyYT YouTube embed
1424 matches = re.findall(
1425 r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
1426 if matches:
1427 return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
1428
1429 # Look for embedded Dailymotion player
1430 matches = re.findall(
1431 r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
1432 if matches:
1433 return _playlist_from_matches(
1434 matches, lambda m: unescapeHTML(m[1]))
1435
1436 # Look for embedded Dailymotion playlist player (#3822)
1437 m = re.search(
1438 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
1439 if m:
1440 playlists = re.findall(
1441 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
1442 if playlists:
1443 return _playlist_from_matches(
1444 playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
1445
1446 # Look for embedded Wistia player
1447 match = re.search(
1448 r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
1449 if match:
1450 embed_url = self._proto_relative_url(
1451 unescapeHTML(match.group('url')))
1452 return {
1453 '_type': 'url_transparent',
1454 'url': embed_url,
1455 'ie_key': 'Wistia',
1456 'uploader': video_uploader,
1457 'title': video_title,
1458 'id': video_id,
1459 }
1460
1461 match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
1462 if match:
1463 return {
1464 '_type': 'url_transparent',
1465 'url': 'http://fast.wistia.net/embed/iframe/{0:}'.format(match.group('id')),
1466 'ie_key': 'Wistia',
1467 'uploader': video_uploader,
1468 'title': video_title,
1469 'id': match.group('id')
1470 }
1471
1472 # Look for SVT player
1473 svt_url = SVTIE._extract_url(webpage)
1474 if svt_url:
1475 return self.url_result(svt_url, 'SVT')
1476
1477 # Look for embedded condenast player
1478 matches = re.findall(
1479 r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
1480 webpage)
1481 if matches:
1482 return {
1483 '_type': 'playlist',
1484 'entries': [{
1485 '_type': 'url',
1486 'ie_key': 'CondeNast',
1487 'url': ma,
1488 } for ma in matches],
1489 'title': video_title,
1490 'id': video_id,
1491 }
1492
1493 # Look for Bandcamp pages with custom domain
1494 mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
1495 if mobj is not None:
1496 burl = unescapeHTML(mobj.group(1))
1497 # Don't set the extractor because it can be a track url or an album
1498 return self.url_result(burl)
1499
1500 # Look for embedded Vevo player
1501 mobj = re.search(
1502 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
1503 if mobj is not None:
1504 return self.url_result(mobj.group('url'))
1505
1506 # Look for embedded Viddler player
1507 mobj = re.search(
1508 r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
1509 webpage)
1510 if mobj is not None:
1511 return self.url_result(mobj.group('url'))
1512
1513 # Look for NYTimes player
1514 mobj = re.search(
1515 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
1516 webpage)
1517 if mobj is not None:
1518 return self.url_result(mobj.group('url'))
1519
1520 # Look for Libsyn player
1521 mobj = re.search(
1522 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
1523 if mobj is not None:
1524 return self.url_result(mobj.group('url'))
1525
1526 # Look for Ooyala videos
1527 mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
1528 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
1529 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
1530 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
1531 if mobj is not None:
1532 return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
1533
1534 # Look for multiple Ooyala embeds on SBN network websites
1535 mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
1536 if mobj is not None:
1537 embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
1538 if embeds:
1539 return _playlist_from_matches(
1540 embeds, getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
1541
1542 # Look for Aparat videos
1543 mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
1544 if mobj is not None:
1545 return self.url_result(mobj.group(1), 'Aparat')
1546
1547 # Look for MPORA videos
1548 mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
1549 if mobj is not None:
1550 return self.url_result(mobj.group(1), 'Mpora')
1551
1552 # Look for embedded NovaMov-based player
1553 mobj = re.search(
1554 r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
1555 (?P<url>http://(?:(?:embed|www)\.)?
1556 (?:novamov\.com|
1557 nowvideo\.(?:ch|sx|eu|at|ag|co)|
1558 videoweed\.(?:es|com)|
1559 movshare\.(?:net|sx|ag)|
1560 divxstage\.(?:eu|net|ch|co|at|ag))
1561 /embed\.php.+?)\1''', webpage)
1562 if mobj is not None:
1563 return self.url_result(mobj.group('url'))
1564
1565 # Look for embedded Facebook player
1566 mobj = re.search(
1567 r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
1568 if mobj is not None:
1569 return self.url_result(mobj.group('url'), 'Facebook')
1570
1571 # Look for embedded VK player
1572 mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
1573 if mobj is not None:
1574 return self.url_result(mobj.group('url'), 'VK')
1575
1576 # Look for embedded Odnoklassniki player
1577 mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:odnoklassniki|ok)\.ru/videoembed/.+?)\1', webpage)
1578 if mobj is not None:
1579 return self.url_result(mobj.group('url'), 'Odnoklassniki')
1580
1581 # Look for embedded ivi player
1582 mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
1583 if mobj is not None:
1584 return self.url_result(mobj.group('url'), 'Ivi')
1585
1586 # Look for embedded Huffington Post player
1587 mobj = re.search(
1588 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
1589 if mobj is not None:
1590 return self.url_result(mobj.group('url'), 'HuffPost')
1591
1592 # Look for embed.ly
1593 mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
1594 if mobj is not None:
1595 return self.url_result(mobj.group('url'))
1596 mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
1597 if mobj is not None:
1598 return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
1599
1600 # Look for funnyordie embed
1601 matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
1602 if matches:
1603 return _playlist_from_matches(
1604 matches, getter=unescapeHTML, ie='FunnyOrDie')
1605
1606 # Look for BBC iPlayer embed
1607 matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
1608 if matches:
1609 return _playlist_from_matches(matches, ie='BBCCoUk')
1610
1611 # Look for embedded RUTV player
1612 rutv_url = RUTVIE._extract_url(webpage)
1613 if rutv_url:
1614 return self.url_result(rutv_url, 'RUTV')
1615
1616 # Look for embedded TVC player
1617 tvc_url = TVCIE._extract_url(webpage)
1618 if tvc_url:
1619 return self.url_result(tvc_url, 'TVC')
1620
1621 # Look for embedded SportBox player
1622 sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
1623 if sportbox_urls:
1624 return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
1625
1626 # Look for embedded PornHub player
1627 pornhub_url = PornHubIE._extract_url(webpage)
1628 if pornhub_url:
1629 return self.url_result(pornhub_url, 'PornHub')
1630
1631 # Look for embedded XHamster player
1632 xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
1633 if xhamster_urls:
1634 return _playlist_from_matches(xhamster_urls, ie='XHamsterEmbed')
1635
1636 # Look for embedded Tvigle player
1637 mobj = re.search(
1638 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
1639 if mobj is not None:
1640 return self.url_result(mobj.group('url'), 'Tvigle')
1641
1642 # Look for embedded TED player
1643 mobj = re.search(
1644 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
1645 if mobj is not None:
1646 return self.url_result(mobj.group('url'), 'TED')
1647
1648 # Look for embedded Ustream videos
1649 mobj = re.search(
1650 r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
1651 if mobj is not None:
1652 return self.url_result(mobj.group('url'), 'Ustream')
1653
1654 # Look for embedded arte.tv player
1655 mobj = re.search(
1656 r'<script [^>]*?src="(?P<url>http://www\.arte\.tv/playerv2/embed[^"]+)"',
1657 webpage)
1658 if mobj is not None:
1659 return self.url_result(mobj.group('url'), 'ArteTVEmbed')
1660
1661 # Look for embedded francetv player
1662 mobj = re.search(
1663 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
1664 webpage)
1665 if mobj is not None:
1666 return self.url_result(mobj.group('url'))
1667
1668 # Look for embedded smotri.com player
1669 smotri_url = SmotriIE._extract_url(webpage)
1670 if smotri_url:
1671 return self.url_result(smotri_url, 'Smotri')
1672
1673 # Look for embedded Myvi.ru player
1674 myvi_url = MyviIE._extract_url(webpage)
1675 if myvi_url:
1676 return self.url_result(myvi_url)
1677
1678 # Look for embedded soundcloud player
1679 mobj = re.search(
1680 r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
1681 webpage)
1682 if mobj is not None:
1683 url = unescapeHTML(mobj.group('url'))
1684 return self.url_result(url)
1685
1686 # Look for embedded vulture.com player
1687 mobj = re.search(
1688 r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
1689 webpage)
1690 if mobj is not None:
1691 url = unescapeHTML(mobj.group('url'))
1692 return self.url_result(url, ie='Vulture')
1693
1694 # Look for embedded mtvservices player
1695 mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
1696 if mtvservices_url:
1697 return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
1698
1699 # Look for embedded yahoo player
1700 mobj = re.search(
1701 r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
1702 webpage)
1703 if mobj is not None:
1704 return self.url_result(mobj.group('url'), 'Yahoo')
1705
1706 # Look for embedded sbs.com.au player
1707 mobj = re.search(
1708 r'''(?x)
1709 (?:
1710 <meta\s+property="og:video"\s+content=|
1711 <iframe[^>]+?src=
1712 )
1713 (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
1714 webpage)
1715 if mobj is not None:
1716 return self.url_result(mobj.group('url'), 'SBS')
1717
1718 # Look for embedded Cinchcast player
1719 mobj = re.search(
1720 r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
1721 webpage)
1722 if mobj is not None:
1723 return self.url_result(mobj.group('url'), 'Cinchcast')
1724
1725 mobj = re.search(
1726 r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
1727 webpage)
1728 if not mobj:
1729 mobj = re.search(
1730 r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
1731 webpage)
1732 if mobj is not None:
1733 return self.url_result(mobj.group('url'), 'MLB')
1734
1735 mobj = re.search(
1736 r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
1737 webpage)
1738 if mobj is not None:
1739 return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
1740
1741 mobj = re.search(
1742 r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
1743 webpage)
1744 if mobj is not None:
1745 return self.url_result(mobj.group('url'), 'Livestream')
1746
1747 # Look for Zapiks embed
1748 mobj = re.search(
1749 r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
1750 if mobj is not None:
1751 return self.url_result(mobj.group('url'), 'Zapiks')
1752
1753 # Look for Kaltura embeds
1754 mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
1755 re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
1756 if mobj is not None:
1757 return self.url_result(smuggle_url(
1758 'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(),
1759 {'source_url': url}), 'Kaltura')
1760
1761 # Look for Eagle.Platform embeds
1762 mobj = re.search(
1763 r'<iframe[^>]+src="(?P<url>https?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage)
1764 if mobj is not None:
1765 return self.url_result(mobj.group('url'), 'EaglePlatform')
1766
1767 # Look for ClipYou (uses Eagle.Platform) embeds
1768 mobj = re.search(
1769 r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
1770 if mobj is not None:
1771 return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
1772
1773 # Look for Pladform embeds
1774 pladform_url = PladformIE._extract_url(webpage)
1775 if pladform_url:
1776 return self.url_result(pladform_url)
1777
1778 # Look for Videomore embeds
1779 videomore_url = VideomoreIE._extract_url(webpage)
1780 if videomore_url:
1781 return self.url_result(videomore_url)
1782
1783 # Look for Playwire embeds
1784 mobj = re.search(
1785 r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
1786 if mobj is not None:
1787 return self.url_result(mobj.group('url'))
1788
1789 # Look for 5min embeds
1790 mobj = re.search(
1791 r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
1792 if mobj is not None:
1793 return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
1794
1795 # Look for Crooks and Liars embeds
1796 mobj = re.search(
1797 r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
1798 if mobj is not None:
1799 return self.url_result(mobj.group('url'))
1800
1801 # Look for NBC Sports VPlayer embeds
1802 nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
1803 if nbc_sports_url:
1804 return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
1805
1806 # Look for Google Drive embeds
1807 google_drive_url = GoogleDriveIE._extract_url(webpage)
1808 if google_drive_url:
1809 return self.url_result(google_drive_url, 'GoogleDrive')
1810
1811 # Look for UDN embeds
1812 mobj = re.search(
1813 r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
1814 if mobj is not None:
1815 return self.url_result(
1816 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
1817
1818 # Look for Senate ISVP iframe
1819 senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
1820 if senate_isvp_url:
1821 return self.url_result(senate_isvp_url, 'SenateISVP')
1822
1823 # Look for Dailymotion Cloud videos
1824 dmcloud_url = DailymotionCloudIE._extract_dmcloud_url(webpage)
1825 if dmcloud_url:
1826 return self.url_result(dmcloud_url, 'DailymotionCloud')
1827
1828 # Look for OnionStudios embeds
1829 onionstudios_url = OnionStudiosIE._extract_url(webpage)
1830 if onionstudios_url:
1831 return self.url_result(onionstudios_url)
1832
1833 # Look for SnagFilms embeds
1834 snagfilms_url = SnagFilmsEmbedIE._extract_url(webpage)
1835 if snagfilms_url:
1836 return self.url_result(snagfilms_url)
1837
1838 # Look for JWPlatform embeds
1839 jwplatform_url = JWPlatformIE._extract_url(webpage)
1840 if jwplatform_url:
1841 return self.url_result(jwplatform_url, 'JWPlatform')
1842
1843 # Look for ScreenwaveMedia embeds
1844 mobj = re.search(ScreenwaveMediaIE.EMBED_PATTERN, webpage)
1845 if mobj is not None:
1846 return self.url_result(unescapeHTML(mobj.group('url')), 'ScreenwaveMedia')
1847
1848 # Look for Digiteka embeds
1849 digiteka_url = DigitekaIE._extract_url(webpage)
1850 if digiteka_url:
1851 return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
1852
1853 # Look for Limelight embeds
1854 mobj = re.search(r'LimelightPlayer\.doLoad(Media|Channel|ChannelList)\(["\'](?P<id>[a-z0-9]{32})', webpage)
1855 if mobj:
1856 lm = {
1857 'Media': 'media',
1858 'Channel': 'channel',
1859 'ChannelList': 'channel_list',
1860 }
1861 return self.url_result('limelight:%s:%s' % (
1862 lm[mobj.group(1)], mobj.group(2)), 'Limelight%s' % mobj.group(1), mobj.group(2))
1863
1864 # Look for AdobeTVVideo embeds
1865 mobj = re.search(
1866 r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
1867 webpage)
1868 if mobj is not None:
1869 return self.url_result(
1870 self._proto_relative_url(unescapeHTML(mobj.group(1))),
1871 'AdobeTVVideo')
1872
1873 def check_video(vurl):
1874 if YoutubeIE.suitable(vurl):
1875 return True
1876 vpath = compat_urlparse.urlparse(vurl).path
1877 vext = determine_ext(vpath)
1878 return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
1879
1880 def filter_video(urls):
1881 return list(filter(check_video, urls))
1882
1883 # Start with something easy: JW Player in SWFObject
1884 found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
1885 if not found:
1886 # Look for gorilla-vid style embedding
1887 found = filter_video(re.findall(r'''(?sx)
1888 (?:
1889 jw_plugins|
1890 JWPlayerOptions|
1891 jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
1892 )
1893 .*?
1894 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
1895 if not found:
1896 # Broaden the search a little bit
1897 found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
1898 if not found:
1899 # Broaden the findall a little bit: JWPlayer JS loader
1900 found = filter_video(re.findall(
1901 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
1902 if not found:
1903 # Flow player
1904 found = filter_video(re.findall(r'''(?xs)
1905 flowplayer\("[^"]+",\s*
1906 \{[^}]+?\}\s*,
1907 \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
1908 ["']?url["']?\s*:\s*["']([^"']+)["']
1909 ''', webpage))
1910 if not found:
1911 # Cinerama player
1912 found = re.findall(
1913 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
1914 if not found:
1915 # Try to find twitter cards info
1916 found = filter_video(re.findall(
1917 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
1918 if not found:
1919 # We look for Open Graph info:
1920 # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
1921 m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
1922 # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
1923 if m_video_type is not None:
1924 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
1925 if not found:
1926 # HTML5 video
1927 found = re.findall(r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
1928 if not found:
1929 REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
1930 found = re.search(
1931 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
1932 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
1933 webpage)
1934 if not found:
1935 # Look also in Refresh HTTP header
1936 refresh_header = head_response.headers.get('Refresh')
1937 if refresh_header:
1938 # In python 2 response HTTP headers are bytestrings
1939 if sys.version_info < (3, 0) and isinstance(refresh_header, str):
1940 refresh_header = refresh_header.decode('iso-8859-1')
1941 found = re.search(REDIRECT_REGEX, refresh_header)
1942 if found:
1943 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
1944 self.report_following_redirect(new_url)
1945 return {
1946 '_type': 'url',
1947 'url': new_url,
1948 }
1949 if not found:
1950 raise UnsupportedError(url)
1951
1952 entries = []
1953 for video_url in found:
1954 video_url = video_url.replace('\\/', '/')
1955 video_url = compat_urlparse.urljoin(url, video_url)
1956 video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
1957
1958 # Sometimes, jwplayer extraction will result in a YouTube URL
1959 if YoutubeIE.suitable(video_url):
1960 entries.append(self.url_result(video_url, 'Youtube'))
1961 continue
1962
1963 # here's a fun little line of code for you:
1964 video_id = os.path.splitext(video_id)[0]
1965
1966 entry_info_dict = {
1967 'id': video_id,
1968 'uploader': video_uploader,
1969 'title': video_title,
1970 'age_limit': age_limit,
1971 }
1972
1973 ext = determine_ext(video_url)
1974 if ext == 'smil':
1975 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
1976 elif ext == 'xspf':
1977 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
1978 elif ext == 'm3u8':
1979 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
1980 elif ext == 'mpd':
1981 entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
1982 else:
1983 entry_info_dict['url'] = video_url
1984
1985 entries.append(entry_info_dict)
1986
1987 if len(entries) == 1:
1988 return entries[0]
1989 else:
1990 for num, e in enumerate(entries, start=1):
1991 # 'url' results don't have a title
1992 if e.get('title') is not None:
1993 e['title'] = '%s (%d)' % (e['title'], num)
1994 return {
1995 '_type': 'playlist',
1996 'entries': entries,
1997 }
Backup clone of https://github.com/ytdl-org unavailable since RIAA fucking raid...