]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/generic.py
Rapsys Git / youtubedl / blob
? search:


96ca398de7ce46599e532a5d1adb4a05ac2fb939
[youtubedl] / youtube_dl / extractor / generic.py
1 # encoding: utf-8
2
3 from __future__ import unicode_literals
4
5 import os
6 import re
7
8 from .common import InfoExtractor
9 from .youtube import YoutubeIE
10 from ..compat import (
11 compat_urllib_parse,
12 compat_urllib_parse_unquote,
13 compat_urllib_request,
14 compat_urlparse,
15 compat_xml_parse_error,
16 )
17 from ..utils import (
18 determine_ext,
19 ExtractorError,
20 float_or_none,
21 HEADRequest,
22 is_html,
23 orderedSet,
24 parse_xml,
25 smuggle_url,
26 unescapeHTML,
27 unified_strdate,
28 unsmuggle_url,
29 UnsupportedError,
30 url_basename,
31 xpath_text,
32 )
33 from .brightcove import BrightcoveIE
34 from .nbc import NBCSportsVPlayerIE
35 from .ooyala import OoyalaIE
36 from .rutv import RUTVIE
37 from .sportbox import SportBoxEmbedIE
38 from .smotri import SmotriIE
39 from .condenast import CondeNastIE
40 from .udn import UDNEmbedIE
41 from .senateisvp import SenateISVPIE
42 from .bliptv import BlipTVIE
43 from .svt import SVTIE
44
45
46 class GenericIE(InfoExtractor):
47 IE_DESC = 'Generic downloader that works on some sites'
48 _VALID_URL = r'.*'
49 IE_NAME = 'generic'
50 _TESTS = [
51 # Direct link to a video
52 {
53 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
54 'md5': '67d406c2bcb6af27fa886f31aa934bbe',
55 'info_dict': {
56 'id': 'trailer',
57 'ext': 'mp4',
58 'title': 'trailer',
59 'upload_date': '20100513',
60 }
61 },
62 # Direct link to media delivered compressed (until Accept-Encoding is *)
63 {
64 'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
65 'md5': '128c42e68b13950268b648275386fc74',
66 'info_dict': {
67 'id': 'FictionJunction-Parallel_Hearts',
68 'ext': 'flac',
69 'title': 'FictionJunction-Parallel_Hearts',
70 'upload_date': '20140522',
71 },
72 'expected_warnings': [
73 'URL could be a direct video link, returning it as such.'
74 ]
75 },
76 # Direct download with broken HEAD
77 {
78 'url': 'http://ai-radio.org:8000/radio.opus',
79 'info_dict': {
80 'id': 'radio',
81 'ext': 'opus',
82 'title': 'radio',
83 },
84 'params': {
85 'skip_download': True, # infinite live stream
86 },
87 'expected_warnings': [
88 r'501.*Not Implemented'
89 ],
90 },
91 # Direct link with incorrect MIME type
92 {
93 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
94 'md5': '4ccbebe5f36706d85221f204d7eb5913',
95 'info_dict': {
96 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
97 'id': '5_Lennart_Poettering_-_Systemd',
98 'ext': 'webm',
99 'title': '5_Lennart_Poettering_-_Systemd',
100 'upload_date': '20141120',
101 },
102 'expected_warnings': [
103 'URL could be a direct video link, returning it as such.'
104 ]
105 },
106 # RSS feed
107 {
108 'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
109 'info_dict': {
110 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
111 'title': 'Zero Punctuation',
112 'description': 're:.*groundbreaking video review series.*'
113 },
114 'playlist_mincount': 11,
115 },
116 # RSS feed with enclosure
117 {
118 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
119 'info_dict': {
120 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
121 'ext': 'm4v',
122 'upload_date': '20150228',
123 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
124 }
125 },
126 # google redirect
127 {
128 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
129 'info_dict': {
130 'id': 'cmQHVoWB5FY',
131 'ext': 'mp4',
132 'upload_date': '20130224',
133 'uploader_id': 'TheVerge',
134 'description': 're:^Chris Ziegler takes a look at the\.*',
135 'uploader': 'The Verge',
136 'title': 'First Firefox OS phones side-by-side',
137 },
138 'params': {
139 'skip_download': False,
140 }
141 },
142 {
143 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
144 'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
145 'info_dict': {
146 'id': '13601338388002',
147 'ext': 'mp4',
148 'uploader': 'www.hodiho.fr',
149 'title': 'R\u00e9gis plante sa Jeep',
150 }
151 },
152 # bandcamp page with custom domain
153 {
154 'add_ie': ['Bandcamp'],
155 'url': 'http://bronyrock.com/track/the-pony-mash',
156 'info_dict': {
157 'id': '3235767654',
158 'ext': 'mp3',
159 'title': 'The Pony Mash',
160 'uploader': 'M_Pallante',
161 },
162 'skip': 'There is a limit of 200 free downloads / month for the test song',
163 },
164 # embedded brightcove video
165 # it also tests brightcove videos that need to set the 'Referer' in the
166 # http requests
167 {
168 'add_ie': ['Brightcove'],
169 'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
170 'info_dict': {
171 'id': '2765128793001',
172 'ext': 'mp4',
173 'title': 'Le cours de bourse : l’analyse technique',
174 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
175 'uploader': 'BFM BUSINESS',
176 },
177 'params': {
178 'skip_download': True,
179 },
180 },
181 {
182 # https://github.com/rg3/youtube-dl/issues/2253
183 'url': 'http://bcove.me/i6nfkrc3',
184 'md5': '0ba9446db037002366bab3b3eb30c88c',
185 'info_dict': {
186 'id': '3101154703001',
187 'ext': 'mp4',
188 'title': 'Still no power',
189 'uploader': 'thestar.com',
190 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
191 },
192 'add_ie': ['Brightcove'],
193 },
194 {
195 'url': 'http://www.championat.com/video/football/v/87/87499.html',
196 'md5': 'fb973ecf6e4a78a67453647444222983',
197 'info_dict': {
198 'id': '3414141473001',
199 'ext': 'mp4',
200 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
201 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
202 'uploader': 'Championat',
203 },
204 },
205 {
206 # https://github.com/rg3/youtube-dl/issues/3541
207 'add_ie': ['Brightcove'],
208 'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
209 'info_dict': {
210 'id': '3866516442001',
211 'ext': 'mp4',
212 'title': 'Leer mij vrouwen kennen: Aflevering 1',
213 'description': 'Leer mij vrouwen kennen: Aflevering 1',
214 'uploader': 'SBS Broadcasting',
215 },
216 'skip': 'Restricted to Netherlands',
217 'params': {
218 'skip_download': True, # m3u8 download
219 },
220 },
221 # ooyala video
222 {
223 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
224 'md5': '166dd577b433b4d4ebfee10b0824d8ff',
225 'info_dict': {
226 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
227 'ext': 'mp4',
228 'title': '2cc213299525360.mov', # that's what we get
229 },
230 'add_ie': ['Ooyala'],
231 },
232 # multiple ooyala embeds on SBN network websites
233 {
234 'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
235 'info_dict': {
236 'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
237 'title': '25 lies you will tell yourself on National Signing Day - SBNation.com',
238 },
239 'playlist_mincount': 3,
240 'params': {
241 'skip_download': True,
242 },
243 'add_ie': ['Ooyala'],
244 },
245 # embed.ly video
246 {
247 'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
248 'info_dict': {
249 'id': '9ODmcdjQcHQ',
250 'ext': 'mp4',
251 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
252 'upload_date': '20140225',
253 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
254 'uploader': 'Tested',
255 'uploader_id': 'testedcom',
256 },
257 # No need to test YoutubeIE here
258 'params': {
259 'skip_download': True,
260 },
261 },
262 # funnyordie embed
263 {
264 'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
265 'info_dict': {
266 'id': '18e820ec3f',
267 'ext': 'mp4',
268 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
269 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
270 },
271 },
272 # BBC iPlayer embeds
273 {
274 'url': 'http://www.bbc.co.uk/blogs/adamcurtis/posts/BUGGER',
275 'info_dict': {
276 'title': 'BBC - Blogs - Adam Curtis - BUGGER',
277 },
278 'playlist_mincount': 18,
279 },
280 # RUTV embed
281 {
282 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
283 'info_dict': {
284 'id': '776940',
285 'ext': 'mp4',
286 'title': 'Охотское море стало целиком российским',
287 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
288 },
289 'params': {
290 # m3u8 download
291 'skip_download': True,
292 },
293 },
294 # SportBox embed
295 {
296 'url': 'http://www.vestifinance.ru/articles/25753',
297 'info_dict': {
298 'id': '25753',
299 'title': 'Вести Экономика ― Прямые трансляции с Форума-выставки "Госзаказ-2013"',
300 },
301 'playlist': [{
302 'info_dict': {
303 'id': '370908',
304 'title': 'Госзаказ. День 3',
305 'ext': 'mp4',
306 }
307 }, {
308 'info_dict': {
309 'id': '370905',
310 'title': 'Госзаказ. День 2',
311 'ext': 'mp4',
312 }
313 }, {
314 'info_dict': {
315 'id': '370902',
316 'title': 'Госзаказ. День 1',
317 'ext': 'mp4',
318 }
319 }],
320 'params': {
321 # m3u8 download
322 'skip_download': True,
323 },
324 },
325 # Embedded TED video
326 {
327 'url': 'http://en.support.wordpress.com/videos/ted-talks/',
328 'md5': '65fdff94098e4a607385a60c5177c638',
329 'info_dict': {
330 'id': '1969',
331 'ext': 'mp4',
332 'title': 'Hidden miracles of the natural world',
333 'uploader': 'Louie Schwartzberg',
334 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
335 }
336 },
337 # Embeded Ustream video
338 {
339 'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
340 'md5': '27b99cdb639c9b12a79bca876a073417',
341 'info_dict': {
342 'id': '45734260',
343 'ext': 'flv',
344 'uploader': 'AU SPA: The NSA and Privacy',
345 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
346 }
347 },
348 # nowvideo embed hidden behind percent encoding
349 {
350 'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
351 'md5': '2baf4ddd70f697d94b1c18cf796d5107',
352 'info_dict': {
353 'id': '06e53103ca9aa',
354 'ext': 'flv',
355 'title': 'Macross Episode 001 Watch Macross Episode 001 onl',
356 'description': 'No description',
357 },
358 },
359 # arte embed
360 {
361 'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
362 'md5': '7653032cbb25bf6c80d80f217055fa43',
363 'info_dict': {
364 'id': '048195-004_PLUS7-F',
365 'ext': 'flv',
366 'title': 'X:enius',
367 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
368 'upload_date': '20140320',
369 },
370 'params': {
371 'skip_download': 'Requires rtmpdump'
372 }
373 },
374 # Condé Nast embed
375 {
376 'url': 'http://www.wired.com/2014/04/honda-asimo/',
377 'md5': 'ba0dfe966fa007657bd1443ee672db0f',
378 'info_dict': {
379 'id': '53501be369702d3275860000',
380 'ext': 'mp4',
381 'title': 'Honda’s New Asimo Robot Is More Human Than Ever',
382 }
383 },
384 # Dailymotion embed
385 {
386 'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
387 'md5': '441aeeb82eb72c422c7f14ec533999cd',
388 'info_dict': {
389 'id': 'k2mm4bCdJ6CQ2i7c8o2',
390 'ext': 'mp4',
391 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
392 'uploader': 'Spi0n',
393 },
394 'add_ie': ['Dailymotion'],
395 },
396 # YouTube embed
397 {
398 'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
399 'info_dict': {
400 'id': 'FXRb4ykk4S0',
401 'ext': 'mp4',
402 'title': 'The NBL Auction 2014',
403 'uploader': 'BADMINTON England',
404 'uploader_id': 'BADMINTONEvents',
405 'upload_date': '20140603',
406 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
407 },
408 'add_ie': ['Youtube'],
409 'params': {
410 'skip_download': True,
411 }
412 },
413 # MTVSercices embed
414 {
415 'url': 'http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too',
416 'md5': '35727f82f58c76d996fc188f9755b0d5',
417 'info_dict': {
418 'id': '0306a69b-8adf-4fb5-aace-75f8e8cbfca9',
419 'ext': 'mp4',
420 'title': 'Review',
421 'description': 'Mario\'s life in the fast lane has never looked so good.',
422 },
423 },
424 # YouTube embed via <data-embed-url="">
425 {
426 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
427 'info_dict': {
428 'id': '4vAffPZIT44',
429 'ext': 'mp4',
430 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
431 'uploader': 'Gameloft',
432 'uploader_id': 'gameloft',
433 'upload_date': '20140828',
434 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
435 },
436 'params': {
437 'skip_download': True,
438 }
439 },
440 # Camtasia studio
441 {
442 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
443 'playlist': [{
444 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
445 'info_dict': {
446 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
447 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
448 'ext': 'flv',
449 'duration': 2235.90,
450 }
451 }, {
452 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
453 'info_dict': {
454 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
455 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
456 'ext': 'flv',
457 'duration': 2235.93,
458 }
459 }],
460 'info_dict': {
461 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
462 }
463 },
464 # Flowplayer
465 {
466 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
467 'md5': '9d65602bf31c6e20014319c7d07fba27',
468 'info_dict': {
469 'id': '5123ea6d5e5a7',
470 'ext': 'mp4',
471 'age_limit': 18,
472 'uploader': 'www.handjobhub.com',
473 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
474 }
475 },
476 # Multiple brightcove videos
477 # https://github.com/rg3/youtube-dl/issues/2283
478 {
479 'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
480 'info_dict': {
481 'id': 'always-never',
482 'title': 'Always / Never - The New Yorker',
483 },
484 'playlist_count': 3,
485 'params': {
486 'extract_flat': False,
487 'skip_download': True,
488 }
489 },
490 # MLB embed
491 {
492 'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
493 'md5': '96f09a37e44da40dd083e12d9a683327',
494 'info_dict': {
495 'id': '33322633',
496 'ext': 'mp4',
497 'title': 'Ump changes call to ball',
498 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
499 'duration': 48,
500 'timestamp': 1401537900,
501 'upload_date': '20140531',
502 'thumbnail': 're:^https?://.*\.jpg$',
503 },
504 },
505 # Wistia embed
506 {
507 'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
508 'md5': '8788b683c777a5cf25621eaf286d0c23',
509 'info_dict': {
510 'id': '1cfaf6b7ea',
511 'ext': 'mov',
512 'title': 'md5:51364a8d3d009997ba99656004b5e20d',
513 'duration': 643.0,
514 'filesize': 182808282,
515 'uploader': 'education-portal.com',
516 },
517 },
518 {
519 'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
520 'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
521 'info_dict': {
522 'id': 'uxjb0lwrcz',
523 'ext': 'mp4',
524 'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
525 'duration': 1715.0,
526 'uploader': 'thoughtworks.wistia.com',
527 },
528 },
529 # Soundcloud embed
530 {
531 'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
532 'info_dict': {
533 'id': '174391317',
534 'ext': 'mp3',
535 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
536 'uploader': 'Sophos Security',
537 'title': 'Chet Chat 171 - Oct 29, 2014',
538 'upload_date': '20141029',
539 }
540 },
541 # Livestream embed
542 {
543 'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
544 'info_dict': {
545 'id': '67864563',
546 'ext': 'flv',
547 'upload_date': '20141112',
548 'title': 'Rosetta #CometLanding webcast HL 10',
549 }
550 },
551 # LazyYT
552 {
553 'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
554 'info_dict': {
555 'id': '1986',
556 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
557 },
558 'playlist_mincount': 2,
559 },
560 # Cinchcast embed
561 {
562 'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
563 'info_dict': {
564 'id': '7141703',
565 'ext': 'mp3',
566 'upload_date': '20141126',
567 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
568 }
569 },
570 # Cinerama player
571 {
572 'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
573 'info_dict': {
574 'id': '730m_DandD_1901_512k',
575 'ext': 'mp4',
576 'uploader': 'www.abc.net.au',
577 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
578 }
579 },
580 # embedded viddler video
581 {
582 'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
583 'info_dict': {
584 'id': '4d03aad9',
585 'ext': 'mp4',
586 'uploader': 'deadspin',
587 'title': 'WALL-TO-GORTAT',
588 'timestamp': 1422285291,
589 'upload_date': '20150126',
590 },
591 'add_ie': ['Viddler'],
592 },
593 # Libsyn embed
594 {
595 'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
596 'info_dict': {
597 'id': '3377616',
598 'ext': 'mp3',
599 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
600 'description': 'md5:601cb790edd05908957dae8aaa866465',
601 'upload_date': '20150220',
602 },
603 },
604 # jwplayer YouTube
605 {
606 'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
607 'info_dict': {
608 'id': 'Mrj4DVp2zeA',
609 'ext': 'mp4',
610 'upload_date': '20150212',
611 'uploader': 'The National Archives UK',
612 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
613 'uploader_id': 'NationalArchives08',
614 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
615 },
616 },
617 # rtl.nl embed
618 {
619 'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
620 'playlist_mincount': 5,
621 'info_dict': {
622 'id': 'aanslagen-kopenhagen',
623 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
624 }
625 },
626 # Zapiks embed
627 {
628 'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
629 'info_dict': {
630 'id': '118046',
631 'ext': 'mp4',
632 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
633 }
634 },
635 # Kaltura embed
636 {
637 'url': 'http://www.monumentalnetwork.com/videos/john-carlson-postgame-2-25-15',
638 'info_dict': {
639 'id': '1_eergr3h1',
640 'ext': 'mp4',
641 'upload_date': '20150226',
642 'uploader_id': 'MonumentalSports-Kaltura@perfectsensedigital.com',
643 'timestamp': int,
644 'title': 'John Carlson Postgame 2/25/15',
645 },
646 },
647 # Eagle.Platform embed (generic URL)
648 {
649 'url': 'http://lenta.ru/news/2015/03/06/navalny/',
650 'info_dict': {
651 'id': '227304',
652 'ext': 'mp4',
653 'title': 'Навальный вышел на свободу',
654 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
655 'thumbnail': 're:^https?://.*\.jpg$',
656 'duration': 87,
657 'view_count': int,
658 'age_limit': 0,
659 },
660 },
661 # ClipYou (Eagle.Platform) embed (custom URL)
662 {
663 'url': 'http://muz-tv.ru/play/7129/',
664 'info_dict': {
665 'id': '12820',
666 'ext': 'mp4',
667 'title': "'O Sole Mio",
668 'thumbnail': 're:^https?://.*\.jpg$',
669 'duration': 216,
670 'view_count': int,
671 },
672 },
673 # Pladform embed
674 {
675 'url': 'http://muz-tv.ru/kinozal/view/7400/',
676 'info_dict': {
677 'id': '100183293',
678 'ext': 'mp4',
679 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
680 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
681 'thumbnail': 're:^https?://.*\.jpg$',
682 'duration': 694,
683 'age_limit': 0,
684 },
685 },
686 # Playwire embed
687 {
688 'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
689 'info_dict': {
690 'id': '3519514',
691 'ext': 'mp4',
692 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
693 'thumbnail': 're:^https?://.*\.png$',
694 'duration': 45.115,
695 },
696 },
697 # 5min embed
698 {
699 'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
700 'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
701 'info_dict': {
702 'id': '518726732',
703 'ext': 'mp4',
704 'title': 'Facebook Creates "On This Day" | Crunch Report',
705 },
706 },
707 # SVT embed
708 {
709 'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
710 'info_dict': {
711 'id': '2900353',
712 'ext': 'flv',
713 'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
714 'duration': 27,
715 'age_limit': 0,
716 },
717 },
718 # Crooks and Liars embed
719 {
720 'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
721 'info_dict': {
722 'id': '8RUoRhRi',
723 'ext': 'mp4',
724 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
725 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
726 'timestamp': 1428207000,
727 'upload_date': '20150405',
728 'uploader': 'Heather',
729 },
730 },
731 # Crooks and Liars external embed
732 {
733 'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
734 'info_dict': {
735 'id': 'MTE3MjUtMzQ2MzA',
736 'ext': 'mp4',
737 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
738 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
739 'timestamp': 1265032391,
740 'upload_date': '20100201',
741 'uploader': 'Heather',
742 },
743 },
744 # NBC Sports vplayer embed
745 {
746 'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
747 'info_dict': {
748 'id': 'ln7x1qSThw4k',
749 'ext': 'flv',
750 'title': "PFT Live: New leader in the 'new-look' defense",
751 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
752 },
753 },
754 # UDN embed
755 {
756 'url': 'http://www.udn.com/news/story/7314/822787',
757 'md5': 'fd2060e988c326991037b9aff9df21a6',
758 'info_dict': {
759 'id': '300346',
760 'ext': 'mp4',
761 'title': '中一中男師變性 全校師生力挺',
762 'thumbnail': 're:^https?://.*\.jpg$',
763 }
764 },
765 # Ooyala embed
766 {
767 'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
768 'info_dict': {
769 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
770 'ext': 'mp4',
771 'description': 'VIDEO: Index/Match versus VLOOKUP.',
772 'title': 'This is what separates the Excel masters from the wannabes',
773 },
774 'params': {
775 # m3u8 downloads
776 'skip_download': True,
777 }
778 },
779 # Contains a SMIL manifest
780 {
781 'url': 'http://www.telewebion.com/fa/1263668/%D9%82%D8%B1%D8%B9%D9%87%E2%80%8C%DA%A9%D8%B4%DB%8C-%D9%84%DB%8C%DA%AF-%D9%82%D9%87%D8%B1%D9%85%D8%A7%D9%86%D8%A7%D9%86-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7/%2B-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84.html',
782 'info_dict': {
783 'id': 'file',
784 'ext': 'flv',
785 'title': '+ Football: Lottery Champions League Europe',
786 'uploader': 'www.telewebion.com',
787 },
788 'params': {
789 # rtmpe downloads
790 'skip_download': True,
791 }
792 }
793 ]
794
795 def report_following_redirect(self, new_url):
796 """Report information extraction."""
797 self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
798
799 def _extract_rss(self, url, video_id, doc):
800 playlist_title = doc.find('./channel/title').text
801 playlist_desc_el = doc.find('./channel/description')
802 playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
803
804 entries = []
805 for it in doc.findall('./channel/item'):
806 next_url = xpath_text(it, 'link', fatal=False)
807 if not next_url:
808 enclosure_nodes = it.findall('./enclosure')
809 for e in enclosure_nodes:
810 next_url = e.attrib.get('url')
811 if next_url:
812 break
813
814 if not next_url:
815 continue
816
817 entries.append({
818 '_type': 'url',
819 'url': next_url,
820 'title': it.find('title').text,
821 })
822
823 return {
824 '_type': 'playlist',
825 'id': url,
826 'title': playlist_title,
827 'description': playlist_desc,
828 'entries': entries,
829 }
830
831 def _extract_camtasia(self, url, video_id, webpage):
832 """ Returns None if no camtasia video can be found. """
833
834 camtasia_cfg = self._search_regex(
835 r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
836 webpage, 'camtasia configuration file', default=None)
837 if camtasia_cfg is None:
838 return None
839
840 title = self._html_search_meta('DC.title', webpage, fatal=True)
841
842 camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
843 camtasia_cfg = self._download_xml(
844 camtasia_url, video_id,
845 note='Downloading camtasia configuration',
846 errnote='Failed to download camtasia configuration')
847 fileset_node = camtasia_cfg.find('./playlist/array/fileset')
848
849 entries = []
850 for n in fileset_node.getchildren():
851 url_n = n.find('./uri')
852 if url_n is None:
853 continue
854
855 entries.append({
856 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
857 'title': '%s - %s' % (title, n.tag),
858 'url': compat_urlparse.urljoin(url, url_n.text),
859 'duration': float_or_none(n.find('./duration').text),
860 })
861
862 return {
863 '_type': 'playlist',
864 'entries': entries,
865 'title': title,
866 }
867
868 def _real_extract(self, url):
869 if url.startswith('//'):
870 return {
871 '_type': 'url',
872 'url': self.http_scheme() + url,
873 }
874
875 parsed_url = compat_urlparse.urlparse(url)
876 if not parsed_url.scheme:
877 default_search = self._downloader.params.get('default_search')
878 if default_search is None:
879 default_search = 'fixup_error'
880
881 if default_search in ('auto', 'auto_warning', 'fixup_error'):
882 if '/' in url:
883 self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
884 return self.url_result('http://' + url)
885 elif default_search != 'fixup_error':
886 if default_search == 'auto_warning':
887 if re.match(r'^(?:url|URL)$', url):
888 raise ExtractorError(
889 'Invalid URL: %r . Call youtube-dl like this: youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url,
890 expected=True)
891 else:
892 self._downloader.report_warning(
893 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url)
894 return self.url_result('ytsearch:' + url)
895
896 if default_search in ('error', 'fixup_error'):
897 raise ExtractorError(
898 '%r is not a valid URL. '
899 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
900 % (url, url), expected=True)
901 else:
902 if ':' not in default_search:
903 default_search += ':'
904 return self.url_result(default_search + url)
905
906 url, smuggled_data = unsmuggle_url(url)
907 force_videoid = None
908 is_intentional = smuggled_data and smuggled_data.get('to_generic')
909 if smuggled_data and 'force_videoid' in smuggled_data:
910 force_videoid = smuggled_data['force_videoid']
911 video_id = force_videoid
912 else:
913 video_id = compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
914
915 self.to_screen('%s: Requesting header' % video_id)
916
917 head_req = HEADRequest(url)
918 head_response = self._request_webpage(
919 head_req, video_id,
920 note=False, errnote='Could not send HEAD request to %s' % url,
921 fatal=False)
922
923 if head_response is not False:
924 # Check for redirect
925 new_url = head_response.geturl()
926 if url != new_url:
927 self.report_following_redirect(new_url)
928 if force_videoid:
929 new_url = smuggle_url(
930 new_url, {'force_videoid': force_videoid})
931 return self.url_result(new_url)
932
933 full_response = None
934 if head_response is False:
935 request = compat_urllib_request.Request(url)
936 request.add_header('Accept-Encoding', '*')
937 full_response = self._request_webpage(request, video_id)
938 head_response = full_response
939
940 # Check for direct link to a video
941 content_type = head_response.headers.get('Content-Type', '')
942 m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
943 if m:
944 upload_date = unified_strdate(
945 head_response.headers.get('Last-Modified'))
946 return {
947 'id': video_id,
948 'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
949 'direct': True,
950 'formats': [{
951 'format_id': m.group('format_id'),
952 'url': url,
953 'vcodec': 'none' if m.group('type') == 'audio' else None
954 }],
955 'upload_date': upload_date,
956 }
957
958 if not self._downloader.params.get('test', False) and not is_intentional:
959 self._downloader.report_warning('Falling back on generic information extractor.')
960
961 if not full_response:
962 request = compat_urllib_request.Request(url)
963 # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
964 # making it impossible to download only chunk of the file (yet we need only 512kB to
965 # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
966 # that will always result in downloading the whole file that is not desirable.
967 # Therefore for extraction pass we have to override Accept-Encoding to any in order
968 # to accept raw bytes and being able to download only a chunk.
969 # It may probably better to solve this by checking Content-Type for application/octet-stream
970 # after HEAD request finishes, but not sure if we can rely on this.
971 request.add_header('Accept-Encoding', '*')
972 full_response = self._request_webpage(request, video_id)
973
974 # Maybe it's a direct link to a video?
975 # Be careful not to download the whole thing!
976 first_bytes = full_response.read(512)
977 if not is_html(first_bytes):
978 self._downloader.report_warning(
979 'URL could be a direct video link, returning it as such.')
980 upload_date = unified_strdate(
981 head_response.headers.get('Last-Modified'))
982 return {
983 'id': video_id,
984 'title': compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0]),
985 'direct': True,
986 'url': url,
987 'upload_date': upload_date,
988 }
989
990 webpage = self._webpage_read_content(
991 full_response, url, video_id, prefix=first_bytes)
992
993 self.report_extraction(video_id)
994
995 # Is it an RSS feed?
996 try:
997 doc = parse_xml(webpage)
998 if doc.tag == 'rss':
999 return self._extract_rss(url, video_id, doc)
1000 except compat_xml_parse_error:
1001 pass
1002
1003 # Is it a Camtasia project?
1004 camtasia_res = self._extract_camtasia(url, video_id, webpage)
1005 if camtasia_res is not None:
1006 return camtasia_res
1007
1008 # Sometimes embedded video player is hidden behind percent encoding
1009 # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
1010 # Unescaping the whole page allows to handle those cases in a generic way
1011 webpage = compat_urllib_parse.unquote(webpage)
1012
1013 # it's tempting to parse this further, but you would
1014 # have to take into account all the variations like
1015 # Video Title - Site Name
1016 # Site Name | Video Title
1017 # Video Title - Tagline | Site Name
1018 # and so on and so forth; it's just not practical
1019 video_title = self._html_search_regex(
1020 r'(?s)<title>(.*?)</title>', webpage, 'video title',
1021 default='video')
1022
1023 # Try to detect age limit automatically
1024 age_limit = self._rta_search(webpage)
1025 # And then there are the jokers who advertise that they use RTA,
1026 # but actually don't.
1027 AGE_LIMIT_MARKERS = [
1028 r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
1029 ]
1030 if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
1031 age_limit = 18
1032
1033 # video uploader is domain name
1034 video_uploader = self._search_regex(
1035 r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
1036
1037 # Helper method
1038 def _playlist_from_matches(matches, getter=None, ie=None):
1039 urlrs = orderedSet(
1040 self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
1041 for m in matches)
1042 return self.playlist_result(
1043 urlrs, playlist_id=video_id, playlist_title=video_title)
1044
1045 # Look for BrightCove:
1046 bc_urls = BrightcoveIE._extract_brightcove_urls(webpage)
1047 if bc_urls:
1048 self.to_screen('Brightcove video detected.')
1049 entries = [{
1050 '_type': 'url',
1051 'url': smuggle_url(bc_url, {'Referer': url}),
1052 'ie_key': 'Brightcove'
1053 } for bc_url in bc_urls]
1054
1055 return {
1056 '_type': 'playlist',
1057 'title': video_title,
1058 'id': video_id,
1059 'entries': entries,
1060 }
1061
1062 # Look for embedded rtl.nl player
1063 matches = re.findall(
1064 r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+video_embed[^"]+)"',
1065 webpage)
1066 if matches:
1067 return _playlist_from_matches(matches, ie='RtlNl')
1068
1069 # Look for embedded (iframe) Vimeo player
1070 mobj = re.search(
1071 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
1072 if mobj:
1073 player_url = unescapeHTML(mobj.group('url'))
1074 surl = smuggle_url(player_url, {'Referer': url})
1075 return self.url_result(surl)
1076 # Look for embedded (swf embed) Vimeo player
1077 mobj = re.search(
1078 r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
1079 if mobj:
1080 return self.url_result(mobj.group(1))
1081
1082 # Look for embedded YouTube player
1083 matches = re.findall(r'''(?x)
1084 (?:
1085 <iframe[^>]+?src=|
1086 data-video-url=|
1087 <embed[^>]+?src=|
1088 embedSWF\(?:\s*|
1089 new\s+SWFObject\(
1090 )
1091 (["\'])
1092 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1093 (?:embed|v|p)/.+?)
1094 \1''', webpage)
1095 if matches:
1096 return _playlist_from_matches(
1097 matches, lambda m: unescapeHTML(m[1]))
1098
1099 # Look for lazyYT YouTube embed
1100 matches = re.findall(
1101 r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
1102 if matches:
1103 return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
1104
1105 # Look for embedded Dailymotion player
1106 matches = re.findall(
1107 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)
1108 if matches:
1109 return _playlist_from_matches(
1110 matches, lambda m: unescapeHTML(m[1]))
1111
1112 # Look for embedded Dailymotion playlist player (#3822)
1113 m = re.search(
1114 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
1115 if m:
1116 playlists = re.findall(
1117 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
1118 if playlists:
1119 return _playlist_from_matches(
1120 playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
1121
1122 # Look for embedded Wistia player
1123 match = re.search(
1124 r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
1125 if match:
1126 embed_url = self._proto_relative_url(
1127 unescapeHTML(match.group('url')))
1128 return {
1129 '_type': 'url_transparent',
1130 'url': embed_url,
1131 'ie_key': 'Wistia',
1132 'uploader': video_uploader,
1133 'title': video_title,
1134 'id': video_id,
1135 }
1136
1137 match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
1138 if match:
1139 return {
1140 '_type': 'url_transparent',
1141 'url': 'http://fast.wistia.net/embed/iframe/{0:}'.format(match.group('id')),
1142 'ie_key': 'Wistia',
1143 'uploader': video_uploader,
1144 'title': video_title,
1145 'id': match.group('id')
1146 }
1147
1148 # Look for embedded blip.tv player
1149 bliptv_url = BlipTVIE._extract_url(webpage)
1150 if bliptv_url:
1151 return self.url_result(bliptv_url, 'BlipTV')
1152
1153 # Look for SVT player
1154 svt_url = SVTIE._extract_url(webpage)
1155 if svt_url:
1156 return self.url_result(svt_url, 'SVT')
1157
1158 # Look for embedded condenast player
1159 matches = re.findall(
1160 r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
1161 webpage)
1162 if matches:
1163 return {
1164 '_type': 'playlist',
1165 'entries': [{
1166 '_type': 'url',
1167 'ie_key': 'CondeNast',
1168 'url': ma,
1169 } for ma in matches],
1170 'title': video_title,
1171 'id': video_id,
1172 }
1173
1174 # Look for Bandcamp pages with custom domain
1175 mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
1176 if mobj is not None:
1177 burl = unescapeHTML(mobj.group(1))
1178 # Don't set the extractor because it can be a track url or an album
1179 return self.url_result(burl)
1180
1181 # Look for embedded Vevo player
1182 mobj = re.search(
1183 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
1184 if mobj is not None:
1185 return self.url_result(mobj.group('url'))
1186
1187 # Look for embedded Viddler player
1188 mobj = re.search(
1189 r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
1190 webpage)
1191 if mobj is not None:
1192 return self.url_result(mobj.group('url'))
1193
1194 # Look for NYTimes player
1195 mobj = re.search(
1196 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
1197 webpage)
1198 if mobj is not None:
1199 return self.url_result(mobj.group('url'))
1200
1201 # Look for Libsyn player
1202 mobj = re.search(
1203 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
1204 if mobj is not None:
1205 return self.url_result(mobj.group('url'))
1206
1207 # Look for Ooyala videos
1208 mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
1209 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
1210 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
1211 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
1212 if mobj is not None:
1213 return OoyalaIE._build_url_result(mobj.group('ec'))
1214
1215 # Look for multiple Ooyala embeds on SBN network websites
1216 mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
1217 if mobj is not None:
1218 embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
1219 if embeds:
1220 return _playlist_from_matches(
1221 embeds, getter=lambda v: OoyalaIE._url_for_embed_code(v['provider_video_id']), ie='Ooyala')
1222
1223 # Look for Aparat videos
1224 mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
1225 if mobj is not None:
1226 return self.url_result(mobj.group(1), 'Aparat')
1227
1228 # Look for MPORA videos
1229 mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
1230 if mobj is not None:
1231 return self.url_result(mobj.group(1), 'Mpora')
1232
1233 # Look for embedded NovaMov-based player
1234 mobj = re.search(
1235 r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
1236 (?P<url>http://(?:(?:embed|www)\.)?
1237 (?:novamov\.com|
1238 nowvideo\.(?:ch|sx|eu|at|ag|co)|
1239 videoweed\.(?:es|com)|
1240 movshare\.(?:net|sx|ag)|
1241 divxstage\.(?:eu|net|ch|co|at|ag))
1242 /embed\.php.+?)\1''', webpage)
1243 if mobj is not None:
1244 return self.url_result(mobj.group('url'))
1245
1246 # Look for embedded Facebook player
1247 mobj = re.search(
1248 r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
1249 if mobj is not None:
1250 return self.url_result(mobj.group('url'), 'Facebook')
1251
1252 # Look for embedded VK player
1253 mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
1254 if mobj is not None:
1255 return self.url_result(mobj.group('url'), 'VK')
1256
1257 # Look for embedded ivi player
1258 mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
1259 if mobj is not None:
1260 return self.url_result(mobj.group('url'), 'Ivi')
1261
1262 # Look for embedded Huffington Post player
1263 mobj = re.search(
1264 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
1265 if mobj is not None:
1266 return self.url_result(mobj.group('url'), 'HuffPost')
1267
1268 # Look for embed.ly
1269 mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
1270 if mobj is not None:
1271 return self.url_result(mobj.group('url'))
1272 mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
1273 if mobj is not None:
1274 return self.url_result(compat_urllib_parse.unquote(mobj.group('url')))
1275
1276 # Look for funnyordie embed
1277 matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
1278 if matches:
1279 return _playlist_from_matches(
1280 matches, getter=unescapeHTML, ie='FunnyOrDie')
1281
1282 # Look for BBC iPlayer embed
1283 matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
1284 if matches:
1285 return _playlist_from_matches(matches, ie='BBCCoUk')
1286
1287 # Look for embedded RUTV player
1288 rutv_url = RUTVIE._extract_url(webpage)
1289 if rutv_url:
1290 return self.url_result(rutv_url, 'RUTV')
1291
1292 # Look for embedded SportBox player
1293 sportbox_urls = SportBoxEmbedIE._extract_urls(webpage)
1294 if sportbox_urls:
1295 return _playlist_from_matches(sportbox_urls, ie='SportBoxEmbed')
1296
1297 # Look for embedded TED player
1298 mobj = re.search(
1299 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
1300 if mobj is not None:
1301 return self.url_result(mobj.group('url'), 'TED')
1302
1303 # Look for embedded Ustream videos
1304 mobj = re.search(
1305 r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
1306 if mobj is not None:
1307 return self.url_result(mobj.group('url'), 'Ustream')
1308
1309 # Look for embedded arte.tv player
1310 mobj = re.search(
1311 r'<script [^>]*?src="(?P<url>http://www\.arte\.tv/playerv2/embed[^"]+)"',
1312 webpage)
1313 if mobj is not None:
1314 return self.url_result(mobj.group('url'), 'ArteTVEmbed')
1315
1316 # Look for embedded smotri.com player
1317 smotri_url = SmotriIE._extract_url(webpage)
1318 if smotri_url:
1319 return self.url_result(smotri_url, 'Smotri')
1320
1321 # Look for embeded soundcloud player
1322 mobj = re.search(
1323 r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
1324 webpage)
1325 if mobj is not None:
1326 url = unescapeHTML(mobj.group('url'))
1327 return self.url_result(url)
1328
1329 # Look for embedded vulture.com player
1330 mobj = re.search(
1331 r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
1332 webpage)
1333 if mobj is not None:
1334 url = unescapeHTML(mobj.group('url'))
1335 return self.url_result(url, ie='Vulture')
1336
1337 # Look for embedded mtvservices player
1338 mobj = re.search(
1339 r'<iframe src="(?P<url>https?://media\.mtvnservices\.com/embed/[^"]+)"',
1340 webpage)
1341 if mobj is not None:
1342 url = unescapeHTML(mobj.group('url'))
1343 return self.url_result(url, ie='MTVServicesEmbedded')
1344
1345 # Look for embedded yahoo player
1346 mobj = re.search(
1347 r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
1348 webpage)
1349 if mobj is not None:
1350 return self.url_result(mobj.group('url'), 'Yahoo')
1351
1352 # Look for embedded sbs.com.au player
1353 mobj = re.search(
1354 r'''(?x)
1355 (?:
1356 <meta\s+property="og:video"\s+content=|
1357 <iframe[^>]+?src=
1358 )
1359 (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
1360 webpage)
1361 if mobj is not None:
1362 return self.url_result(mobj.group('url'), 'SBS')
1363
1364 # Look for embedded Cinchcast player
1365 mobj = re.search(
1366 r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
1367 webpage)
1368 if mobj is not None:
1369 return self.url_result(mobj.group('url'), 'Cinchcast')
1370
1371 mobj = re.search(
1372 r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
1373 webpage)
1374 if not mobj:
1375 mobj = re.search(
1376 r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
1377 webpage)
1378 if mobj is not None:
1379 return self.url_result(mobj.group('url'), 'MLB')
1380
1381 mobj = re.search(
1382 r'<iframe[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
1383 webpage)
1384 if mobj is not None:
1385 return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
1386
1387 mobj = re.search(
1388 r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
1389 webpage)
1390 if mobj is not None:
1391 return self.url_result(mobj.group('url'), 'Livestream')
1392
1393 # Look for Zapiks embed
1394 mobj = re.search(
1395 r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
1396 if mobj is not None:
1397 return self.url_result(mobj.group('url'), 'Zapiks')
1398
1399 # Look for Kaltura embeds
1400 mobj = re.search(
1401 r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage)
1402 if mobj is not None:
1403 return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
1404
1405 # Look for Eagle.Platform embeds
1406 mobj = re.search(
1407 r'<iframe[^>]+src="(?P<url>https?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage)
1408 if mobj is not None:
1409 return self.url_result(mobj.group('url'), 'EaglePlatform')
1410
1411 # Look for ClipYou (uses Eagle.Platform) embeds
1412 mobj = re.search(
1413 r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
1414 if mobj is not None:
1415 return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
1416
1417 # Look for Pladform embeds
1418 mobj = re.search(
1419 r'<iframe[^>]+src="(?P<url>https?://out\.pladform\.ru/player\?.+?)"', webpage)
1420 if mobj is not None:
1421 return self.url_result(mobj.group('url'), 'Pladform')
1422
1423 # Look for Playwire embeds
1424 mobj = re.search(
1425 r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
1426 if mobj is not None:
1427 return self.url_result(mobj.group('url'))
1428
1429 # Look for 5min embeds
1430 mobj = re.search(
1431 r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
1432 if mobj is not None:
1433 return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
1434
1435 # Look for Crooks and Liars embeds
1436 mobj = re.search(
1437 r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
1438 if mobj is not None:
1439 return self.url_result(mobj.group('url'))
1440
1441 # Look for NBC Sports VPlayer embeds
1442 nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
1443 if nbc_sports_url:
1444 return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
1445
1446 # Look for UDN embeds
1447 mobj = re.search(
1448 r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)
1449 if mobj is not None:
1450 return self.url_result(
1451 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
1452
1453 # Look for Senate ISVP iframe
1454 senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
1455 if senate_isvp_url:
1456 return self.url_result(senate_isvp_url, 'SenateISVP')
1457
1458 def check_video(vurl):
1459 if YoutubeIE.suitable(vurl):
1460 return True
1461 vpath = compat_urlparse.urlparse(vurl).path
1462 vext = determine_ext(vpath)
1463 return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
1464
1465 def filter_video(urls):
1466 return list(filter(check_video, urls))
1467
1468 # Start with something easy: JW Player in SWFObject
1469 found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
1470 if not found:
1471 # Look for gorilla-vid style embedding
1472 found = filter_video(re.findall(r'''(?sx)
1473 (?:
1474 jw_plugins|
1475 JWPlayerOptions|
1476 jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
1477 )
1478 .*?
1479 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
1480 if not found:
1481 # Broaden the search a little bit
1482 found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
1483 if not found:
1484 # Broaden the findall a little bit: JWPlayer JS loader
1485 found = filter_video(re.findall(
1486 r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
1487 if not found:
1488 # Flow player
1489 found = filter_video(re.findall(r'''(?xs)
1490 flowplayer\("[^"]+",\s*
1491 \{[^}]+?\}\s*,
1492 \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
1493 ["']?url["']?\s*:\s*["']([^"']+)["']
1494 ''', webpage))
1495 if not found:
1496 # Cinerama player
1497 found = re.findall(
1498 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
1499 if not found:
1500 # Try to find twitter cards info
1501 found = filter_video(re.findall(
1502 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
1503 if not found:
1504 # We look for Open Graph info:
1505 # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
1506 m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
1507 # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
1508 if m_video_type is not None:
1509 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
1510 if not found:
1511 # HTML5 video
1512 found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
1513 if not found:
1514 REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
1515 found = re.search(
1516 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
1517 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
1518 webpage)
1519 if not found:
1520 # Look also in Refresh HTTP header
1521 refresh_header = head_response.headers.get('Refresh')
1522 if refresh_header:
1523 found = re.search(REDIRECT_REGEX, refresh_header)
1524 if found:
1525 new_url = compat_urlparse.urljoin(url, found.group(1))
1526 self.report_following_redirect(new_url)
1527 return {
1528 '_type': 'url',
1529 'url': new_url,
1530 }
1531 if not found:
1532 raise UnsupportedError(url)
1533
1534 entries = []
1535 for video_url in found:
1536 video_url = compat_urlparse.urljoin(url, video_url)
1537 video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
1538
1539 # Sometimes, jwplayer extraction will result in a YouTube URL
1540 if YoutubeIE.suitable(video_url):
1541 entries.append(self.url_result(video_url, 'Youtube'))
1542 continue
1543
1544 # here's a fun little line of code for you:
1545 video_id = os.path.splitext(video_id)[0]
1546
1547 if determine_ext(video_url) == 'smil':
1548 entries.append({
1549 'id': video_id,
1550 'formats': self._extract_smil_formats(video_url, video_id),
1551 'uploader': video_uploader,
1552 'title': video_title,
1553 'age_limit': age_limit,
1554 })
1555 else:
1556 entries.append({
1557 'id': video_id,
1558 'url': video_url,
1559 'uploader': video_uploader,
1560 'title': video_title,
1561 'age_limit': age_limit,
1562 })
1563
1564 if len(entries) == 1:
1565 return entries[0]
1566 else:
1567 for num, e in enumerate(entries, start=1):
1568 # 'url' results don't have a title
1569 if e.get('title') is not None:
1570 e['title'] = '%s (%d)' % (e['title'], num)
1571 return {
1572 '_type': 'playlist',
1573 'entries': entries,
1574 }
Backup clone of https://github.com/ytdl-org unavailable since RIAA fucking raid...