]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/generic.py
Rapsys Git / youtubedl / blob
? search:


3d756e8481e0aba09d2290cfd5a4a8b21369aa7d
[youtubedl] / youtube_dl / extractor / generic.py
1 # encoding: utf-8
2
3 from __future__ import unicode_literals
4
5 import os
6 import re
7
8 from .common import InfoExtractor
9 from .youtube import YoutubeIE
10 from ..compat import (
11 compat_urllib_parse,
12 compat_urlparse,
13 compat_xml_parse_error,
14 )
15 from ..utils import (
16 determine_ext,
17 ExtractorError,
18 float_or_none,
19 HEADRequest,
20 is_html,
21 orderedSet,
22 parse_xml,
23 smuggle_url,
24 unescapeHTML,
25 unified_strdate,
26 unsmuggle_url,
27 UnsupportedError,
28 url_basename,
29 xpath_text,
30 )
31 from .brightcove import BrightcoveIE
32 from .nbc import NBCSportsVPlayerIE
33 from .ooyala import OoyalaIE
34 from .rutv import RUTVIE
35 from .smotri import SmotriIE
36 from .condenast import CondeNastIE
37 from .udn import UDNEmbedIE
38 from .senateisvp import SenateISVPIE
39 from .bliptv import BlipTVIE
40 from .svt import SVTIE
41
42
43 class GenericIE(InfoExtractor):
44 IE_DESC = 'Generic downloader that works on some sites'
45 _VALID_URL = r'.*'
46 IE_NAME = 'generic'
47 _TESTS = [
48 {
49 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
50 'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
51 'info_dict': {
52 'id': '13601338388002',
53 'ext': 'mp4',
54 'uploader': 'www.hodiho.fr',
55 'title': 'R\u00e9gis plante sa Jeep',
56 }
57 },
58 # bandcamp page with custom domain
59 {
60 'add_ie': ['Bandcamp'],
61 'url': 'http://bronyrock.com/track/the-pony-mash',
62 'info_dict': {
63 'id': '3235767654',
64 'ext': 'mp3',
65 'title': 'The Pony Mash',
66 'uploader': 'M_Pallante',
67 },
68 'skip': 'There is a limit of 200 free downloads / month for the test song',
69 },
70 # embedded brightcove video
71 # it also tests brightcove videos that need to set the 'Referer' in the
72 # http requests
73 {
74 'add_ie': ['Brightcove'],
75 'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
76 'info_dict': {
77 'id': '2765128793001',
78 'ext': 'mp4',
79 'title': 'Le cours de bourse : l’analyse technique',
80 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
81 'uploader': 'BFM BUSINESS',
82 },
83 'params': {
84 'skip_download': True,
85 },
86 },
87 {
88 # https://github.com/rg3/youtube-dl/issues/2253
89 'url': 'http://bcove.me/i6nfkrc3',
90 'md5': '0ba9446db037002366bab3b3eb30c88c',
91 'info_dict': {
92 'id': '3101154703001',
93 'ext': 'mp4',
94 'title': 'Still no power',
95 'uploader': 'thestar.com',
96 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
97 },
98 'add_ie': ['Brightcove'],
99 },
100 {
101 'url': 'http://www.championat.com/video/football/v/87/87499.html',
102 'md5': 'fb973ecf6e4a78a67453647444222983',
103 'info_dict': {
104 'id': '3414141473001',
105 'ext': 'mp4',
106 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
107 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
108 'uploader': 'Championat',
109 },
110 },
111 {
112 # https://github.com/rg3/youtube-dl/issues/3541
113 'add_ie': ['Brightcove'],
114 'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
115 'info_dict': {
116 'id': '3866516442001',
117 'ext': 'mp4',
118 'title': 'Leer mij vrouwen kennen: Aflevering 1',
119 'description': 'Leer mij vrouwen kennen: Aflevering 1',
120 'uploader': 'SBS Broadcasting',
121 },
122 'skip': 'Restricted to Netherlands',
123 'params': {
124 'skip_download': True, # m3u8 download
125 },
126 },
127 # Direct link to a video
128 {
129 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
130 'md5': '67d406c2bcb6af27fa886f31aa934bbe',
131 'info_dict': {
132 'id': 'trailer',
133 'ext': 'mp4',
134 'title': 'trailer',
135 'upload_date': '20100513',
136 }
137 },
138 # ooyala video
139 {
140 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
141 'md5': '166dd577b433b4d4ebfee10b0824d8ff',
142 'info_dict': {
143 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
144 'ext': 'mp4',
145 'title': '2cc213299525360.mov', # that's what we get
146 },
147 'add_ie': ['Ooyala'],
148 },
149 # multiple ooyala embeds on SBN network websites
150 {
151 'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
152 'info_dict': {
153 'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
154 'title': '25 lies you will tell yourself on National Signing Day - SBNation.com',
155 },
156 'playlist_mincount': 3,
157 'params': {
158 'skip_download': True,
159 },
160 'add_ie': ['Ooyala'],
161 },
162 # google redirect
163 {
164 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
165 'info_dict': {
166 'id': 'cmQHVoWB5FY',
167 'ext': 'mp4',
168 'upload_date': '20130224',
169 'uploader_id': 'TheVerge',
170 'description': 're:^Chris Ziegler takes a look at the\.*',
171 'uploader': 'The Verge',
172 'title': 'First Firefox OS phones side-by-side',
173 },
174 'params': {
175 'skip_download': False,
176 }
177 },
178 # embed.ly video
179 {
180 'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
181 'info_dict': {
182 'id': '9ODmcdjQcHQ',
183 'ext': 'mp4',
184 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
185 'upload_date': '20140225',
186 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
187 'uploader': 'Tested',
188 'uploader_id': 'testedcom',
189 },
190 # No need to test YoutubeIE here
191 'params': {
192 'skip_download': True,
193 },
194 },
195 # funnyordie embed
196 {
197 'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
198 'info_dict': {
199 'id': '18e820ec3f',
200 'ext': 'mp4',
201 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
202 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
203 },
204 },
205 # BBC iPlayer embeds
206 {
207 'url': 'http://www.bbc.co.uk/blogs/adamcurtis/posts/BUGGER',
208 'info_dict': {
209 'title': 'BBC - Blogs - Adam Curtis - BUGGER',
210 },
211 'playlist_mincount': 18,
212 },
213 # RUTV embed
214 {
215 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
216 'info_dict': {
217 'id': '776940',
218 'ext': 'mp4',
219 'title': 'Охотское море стало целиком российским',
220 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
221 },
222 'params': {
223 # m3u8 download
224 'skip_download': True,
225 },
226 },
227 # Embedded TED video
228 {
229 'url': 'http://en.support.wordpress.com/videos/ted-talks/',
230 'md5': '65fdff94098e4a607385a60c5177c638',
231 'info_dict': {
232 'id': '1969',
233 'ext': 'mp4',
234 'title': 'Hidden miracles of the natural world',
235 'uploader': 'Louie Schwartzberg',
236 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
237 }
238 },
239 # Embeded Ustream video
240 {
241 'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
242 'md5': '27b99cdb639c9b12a79bca876a073417',
243 'info_dict': {
244 'id': '45734260',
245 'ext': 'flv',
246 'uploader': 'AU SPA: The NSA and Privacy',
247 'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
248 }
249 },
250 # nowvideo embed hidden behind percent encoding
251 {
252 'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
253 'md5': '2baf4ddd70f697d94b1c18cf796d5107',
254 'info_dict': {
255 'id': '06e53103ca9aa',
256 'ext': 'flv',
257 'title': 'Macross Episode 001 Watch Macross Episode 001 onl',
258 'description': 'No description',
259 },
260 },
261 # arte embed
262 {
263 'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
264 'md5': '7653032cbb25bf6c80d80f217055fa43',
265 'info_dict': {
266 'id': '048195-004_PLUS7-F',
267 'ext': 'flv',
268 'title': 'X:enius',
269 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
270 'upload_date': '20140320',
271 },
272 'params': {
273 'skip_download': 'Requires rtmpdump'
274 }
275 },
276 # Condé Nast embed
277 {
278 'url': 'http://www.wired.com/2014/04/honda-asimo/',
279 'md5': 'ba0dfe966fa007657bd1443ee672db0f',
280 'info_dict': {
281 'id': '53501be369702d3275860000',
282 'ext': 'mp4',
283 'title': 'Honda’s New Asimo Robot Is More Human Than Ever',
284 }
285 },
286 # Dailymotion embed
287 {
288 'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
289 'md5': '441aeeb82eb72c422c7f14ec533999cd',
290 'info_dict': {
291 'id': 'k2mm4bCdJ6CQ2i7c8o2',
292 'ext': 'mp4',
293 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
294 'uploader': 'Spi0n',
295 },
296 'add_ie': ['Dailymotion'],
297 },
298 # YouTube embed
299 {
300 'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
301 'info_dict': {
302 'id': 'FXRb4ykk4S0',
303 'ext': 'mp4',
304 'title': 'The NBL Auction 2014',
305 'uploader': 'BADMINTON England',
306 'uploader_id': 'BADMINTONEvents',
307 'upload_date': '20140603',
308 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
309 },
310 'add_ie': ['Youtube'],
311 'params': {
312 'skip_download': True,
313 }
314 },
315 # MTVSercices embed
316 {
317 'url': 'http://www.gametrailers.com/news-post/76093/north-america-europe-is-getting-that-mario-kart-8-mercedes-dlc-too',
318 'md5': '35727f82f58c76d996fc188f9755b0d5',
319 'info_dict': {
320 'id': '0306a69b-8adf-4fb5-aace-75f8e8cbfca9',
321 'ext': 'mp4',
322 'title': 'Review',
323 'description': 'Mario\'s life in the fast lane has never looked so good.',
324 },
325 },
326 # YouTube embed via <data-embed-url="">
327 {
328 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
329 'info_dict': {
330 'id': '4vAffPZIT44',
331 'ext': 'mp4',
332 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
333 'uploader': 'Gameloft',
334 'uploader_id': 'gameloft',
335 'upload_date': '20140828',
336 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
337 },
338 'params': {
339 'skip_download': True,
340 }
341 },
342 # Camtasia studio
343 {
344 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
345 'playlist': [{
346 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
347 'info_dict': {
348 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
349 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
350 'ext': 'flv',
351 'duration': 2235.90,
352 }
353 }, {
354 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
355 'info_dict': {
356 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
357 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
358 'ext': 'flv',
359 'duration': 2235.93,
360 }
361 }],
362 'info_dict': {
363 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
364 }
365 },
366 # Flowplayer
367 {
368 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
369 'md5': '9d65602bf31c6e20014319c7d07fba27',
370 'info_dict': {
371 'id': '5123ea6d5e5a7',
372 'ext': 'mp4',
373 'age_limit': 18,
374 'uploader': 'www.handjobhub.com',
375 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
376 }
377 },
378 # RSS feed
379 {
380 'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
381 'info_dict': {
382 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
383 'title': 'Zero Punctuation',
384 'description': 're:.*groundbreaking video review series.*'
385 },
386 'playlist_mincount': 11,
387 },
388 # Multiple brightcove videos
389 # https://github.com/rg3/youtube-dl/issues/2283
390 {
391 'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
392 'info_dict': {
393 'id': 'always-never',
394 'title': 'Always / Never - The New Yorker',
395 },
396 'playlist_count': 3,
397 'params': {
398 'extract_flat': False,
399 'skip_download': True,
400 }
401 },
402 # MLB embed
403 {
404 'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
405 'md5': '96f09a37e44da40dd083e12d9a683327',
406 'info_dict': {
407 'id': '33322633',
408 'ext': 'mp4',
409 'title': 'Ump changes call to ball',
410 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
411 'duration': 48,
412 'timestamp': 1401537900,
413 'upload_date': '20140531',
414 'thumbnail': 're:^https?://.*\.jpg$',
415 },
416 },
417 # Wistia embed
418 {
419 'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
420 'md5': '8788b683c777a5cf25621eaf286d0c23',
421 'info_dict': {
422 'id': '1cfaf6b7ea',
423 'ext': 'mov',
424 'title': 'md5:51364a8d3d009997ba99656004b5e20d',
425 'duration': 643.0,
426 'filesize': 182808282,
427 'uploader': 'education-portal.com',
428 },
429 },
430 {
431 'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
432 'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
433 'info_dict': {
434 'id': 'uxjb0lwrcz',
435 'ext': 'mp4',
436 'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
437 'duration': 1715.0,
438 'uploader': 'thoughtworks.wistia.com',
439 },
440 },
441 # Direct download with broken HEAD
442 {
443 'url': 'http://ai-radio.org:8000/radio.opus',
444 'info_dict': {
445 'id': 'radio',
446 'ext': 'opus',
447 'title': 'radio',
448 },
449 'params': {
450 'skip_download': True, # infinite live stream
451 },
452 'expected_warnings': [
453 r'501.*Not Implemented'
454 ],
455 },
456 # Soundcloud embed
457 {
458 'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
459 'info_dict': {
460 'id': '174391317',
461 'ext': 'mp3',
462 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
463 'uploader': 'Sophos Security',
464 'title': 'Chet Chat 171 - Oct 29, 2014',
465 'upload_date': '20141029',
466 }
467 },
468 # Livestream embed
469 {
470 'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
471 'info_dict': {
472 'id': '67864563',
473 'ext': 'flv',
474 'upload_date': '20141112',
475 'title': 'Rosetta #CometLanding webcast HL 10',
476 }
477 },
478 # LazyYT
479 {
480 'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
481 'info_dict': {
482 'id': '1986',
483 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
484 },
485 'playlist_mincount': 2,
486 },
487 # Direct link with incorrect MIME type
488 {
489 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
490 'md5': '4ccbebe5f36706d85221f204d7eb5913',
491 'info_dict': {
492 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
493 'id': '5_Lennart_Poettering_-_Systemd',
494 'ext': 'webm',
495 'title': '5_Lennart_Poettering_-_Systemd',
496 'upload_date': '20141120',
497 },
498 'expected_warnings': [
499 'URL could be a direct video link, returning it as such.'
500 ]
501 },
502 # Cinchcast embed
503 {
504 'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
505 'info_dict': {
506 'id': '7141703',
507 'ext': 'mp3',
508 'upload_date': '20141126',
509 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
510 }
511 },
512 # Cinerama player
513 {
514 'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
515 'info_dict': {
516 'id': '730m_DandD_1901_512k',
517 'ext': 'mp4',
518 'uploader': 'www.abc.net.au',
519 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
520 }
521 },
522 # embedded viddler video
523 {
524 'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
525 'info_dict': {
526 'id': '4d03aad9',
527 'ext': 'mp4',
528 'uploader': 'deadspin',
529 'title': 'WALL-TO-GORTAT',
530 'timestamp': 1422285291,
531 'upload_date': '20150126',
532 },
533 'add_ie': ['Viddler'],
534 },
535 # Libsyn embed
536 {
537 'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
538 'info_dict': {
539 'id': '3377616',
540 'ext': 'mp3',
541 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
542 'description': 'md5:601cb790edd05908957dae8aaa866465',
543 'upload_date': '20150220',
544 },
545 },
546 # jwplayer YouTube
547 {
548 'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
549 'info_dict': {
550 'id': 'Mrj4DVp2zeA',
551 'ext': 'mp4',
552 'upload_date': '20150212',
553 'uploader': 'The National Archives UK',
554 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
555 'uploader_id': 'NationalArchives08',
556 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
557 },
558 },
559 # rtl.nl embed
560 {
561 'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
562 'playlist_mincount': 5,
563 'info_dict': {
564 'id': 'aanslagen-kopenhagen',
565 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
566 }
567 },
568 # Zapiks embed
569 {
570 'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
571 'info_dict': {
572 'id': '118046',
573 'ext': 'mp4',
574 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
575 }
576 },
577 # Kaltura embed
578 {
579 'url': 'http://www.monumentalnetwork.com/videos/john-carlson-postgame-2-25-15',
580 'info_dict': {
581 'id': '1_eergr3h1',
582 'ext': 'mp4',
583 'upload_date': '20150226',
584 'uploader_id': 'MonumentalSports-Kaltura@perfectsensedigital.com',
585 'timestamp': int,
586 'title': 'John Carlson Postgame 2/25/15',
587 },
588 },
589 # Eagle.Platform embed (generic URL)
590 {
591 'url': 'http://lenta.ru/news/2015/03/06/navalny/',
592 'info_dict': {
593 'id': '227304',
594 'ext': 'mp4',
595 'title': 'Навальный вышел на свободу',
596 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
597 'thumbnail': 're:^https?://.*\.jpg$',
598 'duration': 87,
599 'view_count': int,
600 'age_limit': 0,
601 },
602 },
603 # ClipYou (Eagle.Platform) embed (custom URL)
604 {
605 'url': 'http://muz-tv.ru/play/7129/',
606 'info_dict': {
607 'id': '12820',
608 'ext': 'mp4',
609 'title': "'O Sole Mio",
610 'thumbnail': 're:^https?://.*\.jpg$',
611 'duration': 216,
612 'view_count': int,
613 },
614 },
615 # Pladform embed
616 {
617 'url': 'http://muz-tv.ru/kinozal/view/7400/',
618 'info_dict': {
619 'id': '100183293',
620 'ext': 'mp4',
621 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
622 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
623 'thumbnail': 're:^https?://.*\.jpg$',
624 'duration': 694,
625 'age_limit': 0,
626 },
627 },
628 # Playwire embed
629 {
630 'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
631 'info_dict': {
632 'id': '3519514',
633 'ext': 'mp4',
634 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
635 'thumbnail': 're:^https?://.*\.png$',
636 'duration': 45.115,
637 },
638 },
639 # 5min embed
640 {
641 'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
642 'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
643 'info_dict': {
644 'id': '518726732',
645 'ext': 'mp4',
646 'title': 'Facebook Creates "On This Day" | Crunch Report',
647 },
648 },
649 # SVT embed
650 {
651 'url': 'http://www.svt.se/sport/ishockey/jagr-tacklar-giroux-under-intervjun',
652 'info_dict': {
653 'id': '2900353',
654 'ext': 'flv',
655 'title': 'Här trycker Jagr till Giroux (under SVT-intervjun)',
656 'duration': 27,
657 'age_limit': 0,
658 },
659 },
660 # RSS feed with enclosure
661 {
662 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
663 'info_dict': {
664 'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
665 'ext': 'm4v',
666 'upload_date': '20150228',
667 'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
668 }
669 },
670 # Crooks and Liars embed
671 {
672 'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
673 'info_dict': {
674 'id': '8RUoRhRi',
675 'ext': 'mp4',
676 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
677 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
678 'timestamp': 1428207000,
679 'upload_date': '20150405',
680 'uploader': 'Heather',
681 },
682 },
683 # Crooks and Liars external embed
684 {
685 'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
686 'info_dict': {
687 'id': 'MTE3MjUtMzQ2MzA',
688 'ext': 'mp4',
689 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
690 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
691 'timestamp': 1265032391,
692 'upload_date': '20100201',
693 'uploader': 'Heather',
694 },
695 },
696 # NBC Sports vplayer embed
697 {
698 'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
699 'info_dict': {
700 'id': 'ln7x1qSThw4k',
701 'ext': 'flv',
702 'title': "PFT Live: New leader in the 'new-look' defense",
703 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
704 },
705 },
706 # UDN embed
707 {
708 'url': 'http://www.udn.com/news/story/7314/822787',
709 'md5': 'fd2060e988c326991037b9aff9df21a6',
710 'info_dict': {
711 'id': '300346',
712 'ext': 'mp4',
713 'title': '中一中男師變性 全校師生力挺',
714 'thumbnail': 're:^https?://.*\.jpg$',
715 }
716 },
717 # Ooyala embed
718 {
719 'url': 'http://www.businessinsider.com/excel-index-match-vlookup-video-how-to-2015-2?IR=T',
720 'info_dict': {
721 'id': '50YnY4czr4ms1vJ7yz3xzq0excz_pUMs',
722 'ext': 'mp4',
723 'description': 'VIDEO: Index/Match versus VLOOKUP.',
724 'title': 'This is what separates the Excel masters from the wannabes',
725 },
726 'params': {
727 # m3u8 downloads
728 'skip_download': True,
729 }
730 },
731 # Contains a SMIL manifest
732 {
733 'url': 'http://www.telewebion.com/fa/1263668/%D9%82%D8%B1%D8%B9%D9%87%E2%80%8C%DA%A9%D8%B4%DB%8C-%D9%84%DB%8C%DA%AF-%D9%82%D9%87%D8%B1%D9%85%D8%A7%D9%86%D8%A7%D9%86-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7/%2B-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84.html',
734 'info_dict': {
735 'id': 'file',
736 'ext': 'flv',
737 'title': '+ Football: Lottery Champions League Europe',
738 'uploader': 'www.telewebion.com',
739 },
740 'params': {
741 # rtmpe downloads
742 'skip_download': True,
743 }
744 }
745 ]
746
747 def report_following_redirect(self, new_url):
748 """Report information extraction."""
749 self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
750
751 def _extract_rss(self, url, video_id, doc):
752 playlist_title = doc.find('./channel/title').text
753 playlist_desc_el = doc.find('./channel/description')
754 playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
755
756 entries = []
757 for it in doc.findall('./channel/item'):
758 next_url = xpath_text(it, 'link', fatal=False)
759 if not next_url:
760 enclosure_nodes = it.findall('./enclosure')
761 for e in enclosure_nodes:
762 next_url = e.attrib.get('url')
763 if next_url:
764 break
765
766 if not next_url:
767 continue
768
769 entries.append({
770 '_type': 'url',
771 'url': next_url,
772 'title': it.find('title').text,
773 })
774
775 return {
776 '_type': 'playlist',
777 'id': url,
778 'title': playlist_title,
779 'description': playlist_desc,
780 'entries': entries,
781 }
782
783 def _extract_camtasia(self, url, video_id, webpage):
784 """ Returns None if no camtasia video can be found. """
785
786 camtasia_cfg = self._search_regex(
787 r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
788 webpage, 'camtasia configuration file', default=None)
789 if camtasia_cfg is None:
790 return None
791
792 title = self._html_search_meta('DC.title', webpage, fatal=True)
793
794 camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
795 camtasia_cfg = self._download_xml(
796 camtasia_url, video_id,
797 note='Downloading camtasia configuration',
798 errnote='Failed to download camtasia configuration')
799 fileset_node = camtasia_cfg.find('./playlist/array/fileset')
800
801 entries = []
802 for n in fileset_node.getchildren():
803 url_n = n.find('./uri')
804 if url_n is None:
805 continue
806
807 entries.append({
808 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
809 'title': '%s - %s' % (title, n.tag),
810 'url': compat_urlparse.urljoin(url, url_n.text),
811 'duration': float_or_none(n.find('./duration').text),
812 })
813
814 return {
815 '_type': 'playlist',
816 'entries': entries,
817 'title': title,
818 }
819
820 def _real_extract(self, url):
821 if url.startswith('//'):
822 return {
823 '_type': 'url',
824 'url': self.http_scheme() + url,
825 }
826
827 parsed_url = compat_urlparse.urlparse(url)
828 if not parsed_url.scheme:
829 default_search = self._downloader.params.get('default_search')
830 if default_search is None:
831 default_search = 'fixup_error'
832
833 if default_search in ('auto', 'auto_warning', 'fixup_error'):
834 if '/' in url:
835 self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
836 return self.url_result('http://' + url)
837 elif default_search != 'fixup_error':
838 if default_search == 'auto_warning':
839 if re.match(r'^(?:url|URL)$', url):
840 raise ExtractorError(
841 'Invalid URL: %r . Call youtube-dl like this: youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url,
842 expected=True)
843 else:
844 self._downloader.report_warning(
845 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url)
846 return self.url_result('ytsearch:' + url)
847
848 if default_search in ('error', 'fixup_error'):
849 raise ExtractorError(
850 '%r is not a valid URL. '
851 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
852 % (url, url), expected=True)
853 else:
854 if ':' not in default_search:
855 default_search += ':'
856 return self.url_result(default_search + url)
857
858 url, smuggled_data = unsmuggle_url(url)
859 force_videoid = None
860 is_intentional = smuggled_data and smuggled_data.get('to_generic')
861 if smuggled_data and 'force_videoid' in smuggled_data:
862 force_videoid = smuggled_data['force_videoid']
863 video_id = force_videoid
864 else:
865 video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0]
866
867 self.to_screen('%s: Requesting header' % video_id)
868
869 head_req = HEADRequest(url)
870 head_response = self._request_webpage(
871 head_req, video_id,
872 note=False, errnote='Could not send HEAD request to %s' % url,
873 fatal=False)
874
875 if head_response is not False:
876 # Check for redirect
877 new_url = head_response.geturl()
878 if url != new_url:
879 self.report_following_redirect(new_url)
880 if force_videoid:
881 new_url = smuggle_url(
882 new_url, {'force_videoid': force_videoid})
883 return self.url_result(new_url)
884
885 full_response = None
886 if head_response is False:
887 full_response = self._request_webpage(url, video_id)
888 head_response = full_response
889
890 # Check for direct link to a video
891 content_type = head_response.headers.get('Content-Type', '')
892 m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
893 if m:
894 upload_date = unified_strdate(
895 head_response.headers.get('Last-Modified'))
896 return {
897 'id': video_id,
898 'title': os.path.splitext(url_basename(url))[0],
899 'direct': True,
900 'formats': [{
901 'format_id': m.group('format_id'),
902 'url': url,
903 'vcodec': 'none' if m.group('type') == 'audio' else None
904 }],
905 'upload_date': upload_date,
906 }
907
908 if not self._downloader.params.get('test', False) and not is_intentional:
909 self._downloader.report_warning('Falling back on generic information extractor.')
910
911 if not full_response:
912 full_response = self._request_webpage(url, video_id)
913
914 # Maybe it's a direct link to a video?
915 # Be careful not to download the whole thing!
916 first_bytes = full_response.read(512)
917 if not is_html(first_bytes):
918 self._downloader.report_warning(
919 'URL could be a direct video link, returning it as such.')
920 upload_date = unified_strdate(
921 head_response.headers.get('Last-Modified'))
922 return {
923 'id': video_id,
924 'title': os.path.splitext(url_basename(url))[0],
925 'direct': True,
926 'url': url,
927 'upload_date': upload_date,
928 }
929
930 webpage = self._webpage_read_content(
931 full_response, url, video_id, prefix=first_bytes)
932
933 self.report_extraction(video_id)
934
935 # Is it an RSS feed?
936 try:
937 doc = parse_xml(webpage)
938 if doc.tag == 'rss':
939 return self._extract_rss(url, video_id, doc)
940 except compat_xml_parse_error:
941 pass
942
943 # Is it a Camtasia project?
944 camtasia_res = self._extract_camtasia(url, video_id, webpage)
945 if camtasia_res is not None:
946 return camtasia_res
947
948 # Sometimes embedded video player is hidden behind percent encoding
949 # (e.g. https://github.com/rg3/youtube-dl/issues/2448)
950 # Unescaping the whole page allows to handle those cases in a generic way
951 webpage = compat_urllib_parse.unquote(webpage)
952
953 # it's tempting to parse this further, but you would
954 # have to take into account all the variations like
955 # Video Title - Site Name
956 # Site Name | Video Title
957 # Video Title - Tagline | Site Name
958 # and so on and so forth; it's just not practical
959 video_title = self._html_search_regex(
960 r'(?s)<title>(.*?)</title>', webpage, 'video title',
961 default='video')
962
963 # Try to detect age limit automatically
964 age_limit = self._rta_search(webpage)
965 # And then there are the jokers who advertise that they use RTA,
966 # but actually don't.
967 AGE_LIMIT_MARKERS = [
968 r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>',
969 ]
970 if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
971 age_limit = 18
972
973 # video uploader is domain name
974 video_uploader = self._search_regex(
975 r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
976
977 # Helper method
978 def _playlist_from_matches(matches, getter=None, ie=None):
979 urlrs = orderedSet(
980 self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
981 for m in matches)
982 return self.playlist_result(
983 urlrs, playlist_id=video_id, playlist_title=video_title)
984
985 # Look for BrightCove:
986 bc_urls = BrightcoveIE._extract_brightcove_urls(webpage)
987 if bc_urls:
988 self.to_screen('Brightcove video detected.')
989 entries = [{
990 '_type': 'url',
991 'url': smuggle_url(bc_url, {'Referer': url}),
992 'ie_key': 'Brightcove'
993 } for bc_url in bc_urls]
994
995 return {
996 '_type': 'playlist',
997 'title': video_title,
998 'id': video_id,
999 'entries': entries,
1000 }
1001
1002 # Look for embedded rtl.nl player
1003 matches = re.findall(
1004 r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+video_embed[^"]+)"',
1005 webpage)
1006 if matches:
1007 return _playlist_from_matches(matches, ie='RtlNl')
1008
1009 # Look for embedded (iframe) Vimeo player
1010 mobj = re.search(
1011 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
1012 if mobj:
1013 player_url = unescapeHTML(mobj.group('url'))
1014 surl = smuggle_url(player_url, {'Referer': url})
1015 return self.url_result(surl)
1016 # Look for embedded (swf embed) Vimeo player
1017 mobj = re.search(
1018 r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
1019 if mobj:
1020 return self.url_result(mobj.group(1))
1021
1022 # Look for embedded YouTube player
1023 matches = re.findall(r'''(?x)
1024 (?:
1025 <iframe[^>]+?src=|
1026 data-video-url=|
1027 <embed[^>]+?src=|
1028 embedSWF\(?:\s*|
1029 new\s+SWFObject\(
1030 )
1031 (["\'])
1032 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1033 (?:embed|v|p)/.+?)
1034 \1''', webpage)
1035 if matches:
1036 return _playlist_from_matches(
1037 matches, lambda m: unescapeHTML(m[1]))
1038
1039 # Look for lazyYT YouTube embed
1040 matches = re.findall(
1041 r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
1042 if matches:
1043 return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
1044
1045 # Look for embedded Dailymotion player
1046 matches = re.findall(
1047 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)
1048 if matches:
1049 return _playlist_from_matches(
1050 matches, lambda m: unescapeHTML(m[1]))
1051
1052 # Look for embedded Dailymotion playlist player (#3822)
1053 m = re.search(
1054 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
1055 if m:
1056 playlists = re.findall(
1057 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
1058 if playlists:
1059 return _playlist_from_matches(
1060 playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
1061
1062 # Look for embedded Wistia player
1063 match = re.search(
1064 r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
1065 if match:
1066 embed_url = self._proto_relative_url(
1067 unescapeHTML(match.group('url')))
1068 return {
1069 '_type': 'url_transparent',
1070 'url': embed_url,
1071 'ie_key': 'Wistia',
1072 'uploader': video_uploader,
1073 'title': video_title,
1074 'id': video_id,
1075 }
1076
1077 match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
1078 if match:
1079 return {
1080 '_type': 'url_transparent',
1081 'url': 'http://fast.wistia.net/embed/iframe/{0:}'.format(match.group('id')),
1082 'ie_key': 'Wistia',
1083 'uploader': video_uploader,
1084 'title': video_title,
1085 'id': match.group('id')
1086 }
1087
1088 # Look for embedded blip.tv player
1089 bliptv_url = BlipTVIE._extract_url(webpage)
1090 if bliptv_url:
1091 return self.url_result(bliptv_url, 'BlipTV')
1092
1093 # Look for SVT player
1094 svt_url = SVTIE._extract_url(webpage)
1095 if svt_url:
1096 return self.url_result(svt_url, 'SVT')
1097
1098 # Look for embedded condenast player
1099 matches = re.findall(
1100 r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="(https?://player\.cnevids\.com/embed/[^"]+")',
1101 webpage)
1102 if matches:
1103 return {
1104 '_type': 'playlist',
1105 'entries': [{
1106 '_type': 'url',
1107 'ie_key': 'CondeNast',
1108 'url': ma,
1109 } for ma in matches],
1110 'title': video_title,
1111 'id': video_id,
1112 }
1113
1114 # Look for Bandcamp pages with custom domain
1115 mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
1116 if mobj is not None:
1117 burl = unescapeHTML(mobj.group(1))
1118 # Don't set the extractor because it can be a track url or an album
1119 return self.url_result(burl)
1120
1121 # Look for embedded Vevo player
1122 mobj = re.search(
1123 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
1124 if mobj is not None:
1125 return self.url_result(mobj.group('url'))
1126
1127 # Look for embedded Viddler player
1128 mobj = re.search(
1129 r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
1130 webpage)
1131 if mobj is not None:
1132 return self.url_result(mobj.group('url'))
1133
1134 # Look for NYTimes player
1135 mobj = re.search(
1136 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
1137 webpage)
1138 if mobj is not None:
1139 return self.url_result(mobj.group('url'))
1140
1141 # Look for Libsyn player
1142 mobj = re.search(
1143 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
1144 if mobj is not None:
1145 return self.url_result(mobj.group('url'))
1146
1147 # Look for Ooyala videos
1148 mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
1149 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
1150 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
1151 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
1152 if mobj is not None:
1153 return OoyalaIE._build_url_result(mobj.group('ec'))
1154
1155 # Look for multiple Ooyala embeds on SBN network websites
1156 mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
1157 if mobj is not None:
1158 embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
1159 if embeds:
1160 return _playlist_from_matches(
1161 embeds, getter=lambda v: OoyalaIE._url_for_embed_code(v['provider_video_id']), ie='Ooyala')
1162
1163 # Look for Aparat videos
1164 mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
1165 if mobj is not None:
1166 return self.url_result(mobj.group(1), 'Aparat')
1167
1168 # Look for MPORA videos
1169 mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
1170 if mobj is not None:
1171 return self.url_result(mobj.group(1), 'Mpora')
1172
1173 # Look for embedded NovaMov-based player
1174 mobj = re.search(
1175 r'''(?x)<(?:pagespeed_)?iframe[^>]+?src=(["\'])
1176 (?P<url>http://(?:(?:embed|www)\.)?
1177 (?:novamov\.com|
1178 nowvideo\.(?:ch|sx|eu|at|ag|co)|
1179 videoweed\.(?:es|com)|
1180 movshare\.(?:net|sx|ag)|
1181 divxstage\.(?:eu|net|ch|co|at|ag))
1182 /embed\.php.+?)\1''', webpage)
1183 if mobj is not None:
1184 return self.url_result(mobj.group('url'))
1185
1186 # Look for embedded Facebook player
1187 mobj = re.search(
1188 r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
1189 if mobj is not None:
1190 return self.url_result(mobj.group('url'), 'Facebook')
1191
1192 # Look for embedded VK player
1193 mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
1194 if mobj is not None:
1195 return self.url_result(mobj.group('url'), 'VK')
1196
1197 # Look for embedded ivi player
1198 mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
1199 if mobj is not None:
1200 return self.url_result(mobj.group('url'), 'Ivi')
1201
1202 # Look for embedded Huffington Post player
1203 mobj = re.search(
1204 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
1205 if mobj is not None:
1206 return self.url_result(mobj.group('url'), 'HuffPost')
1207
1208 # Look for embed.ly
1209 mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
1210 if mobj is not None:
1211 return self.url_result(mobj.group('url'))
1212 mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
1213 if mobj is not None:
1214 return self.url_result(compat_urllib_parse.unquote(mobj.group('url')))
1215
1216 # Look for funnyordie embed
1217 matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
1218 if matches:
1219 return _playlist_from_matches(
1220 matches, getter=unescapeHTML, ie='FunnyOrDie')
1221
1222 # Look for BBC iPlayer embed
1223 matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
1224 if matches:
1225 return _playlist_from_matches(matches, ie='BBCCoUk')
1226
1227 # Look for embedded RUTV player
1228 rutv_url = RUTVIE._extract_url(webpage)
1229 if rutv_url:
1230 return self.url_result(rutv_url, 'RUTV')
1231
1232 # Look for embedded TED player
1233 mobj = re.search(
1234 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
1235 if mobj is not None:
1236 return self.url_result(mobj.group('url'), 'TED')
1237
1238 # Look for embedded Ustream videos
1239 mobj = re.search(
1240 r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
1241 if mobj is not None:
1242 return self.url_result(mobj.group('url'), 'Ustream')
1243
1244 # Look for embedded arte.tv player
1245 mobj = re.search(
1246 r'<script [^>]*?src="(?P<url>http://www\.arte\.tv/playerv2/embed[^"]+)"',
1247 webpage)
1248 if mobj is not None:
1249 return self.url_result(mobj.group('url'), 'ArteTVEmbed')
1250
1251 # Look for embedded smotri.com player
1252 smotri_url = SmotriIE._extract_url(webpage)
1253 if smotri_url:
1254 return self.url_result(smotri_url, 'Smotri')
1255
1256 # Look for embeded soundcloud player
1257 mobj = re.search(
1258 r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
1259 webpage)
1260 if mobj is not None:
1261 url = unescapeHTML(mobj.group('url'))
1262 return self.url_result(url)
1263
1264 # Look for embedded vulture.com player
1265 mobj = re.search(
1266 r'<iframe src="(?P<url>https?://video\.vulture\.com/[^"]+)"',
1267 webpage)
1268 if mobj is not None:
1269 url = unescapeHTML(mobj.group('url'))
1270 return self.url_result(url, ie='Vulture')
1271
1272 # Look for embedded mtvservices player
1273 mobj = re.search(
1274 r'<iframe src="(?P<url>https?://media\.mtvnservices\.com/embed/[^"]+)"',
1275 webpage)
1276 if mobj is not None:
1277 url = unescapeHTML(mobj.group('url'))
1278 return self.url_result(url, ie='MTVServicesEmbedded')
1279
1280 # Look for embedded yahoo player
1281 mobj = re.search(
1282 r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
1283 webpage)
1284 if mobj is not None:
1285 return self.url_result(mobj.group('url'), 'Yahoo')
1286
1287 # Look for embedded sbs.com.au player
1288 mobj = re.search(
1289 r'''(?x)
1290 (?:
1291 <meta\s+property="og:video"\s+content=|
1292 <iframe[^>]+?src=
1293 )
1294 (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
1295 webpage)
1296 if mobj is not None:
1297 return self.url_result(mobj.group('url'), 'SBS')
1298
1299 # Look for embedded Cinchcast player
1300 mobj = re.search(
1301 r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
1302 webpage)
1303 if mobj is not None:
1304 return self.url_result(mobj.group('url'), 'Cinchcast')
1305
1306 mobj = re.search(
1307 r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
1308 webpage)
1309 if not mobj:
1310 mobj = re.search(
1311 r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
1312 webpage)
1313 if mobj is not None:
1314 return self.url_result(mobj.group('url'), 'MLB')
1315
1316 mobj = re.search(
1317 r'<iframe[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
1318 webpage)
1319 if mobj is not None:
1320 return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
1321
1322 mobj = re.search(
1323 r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
1324 webpage)
1325 if mobj is not None:
1326 return self.url_result(mobj.group('url'), 'Livestream')
1327
1328 # Look for Zapiks embed
1329 mobj = re.search(
1330 r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
1331 if mobj is not None:
1332 return self.url_result(mobj.group('url'), 'Zapiks')
1333
1334 # Look for Kaltura embeds
1335 mobj = re.search(
1336 r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage)
1337 if mobj is not None:
1338 return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
1339
1340 # Look for Eagle.Platform embeds
1341 mobj = re.search(
1342 r'<iframe[^>]+src="(?P<url>https?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage)
1343 if mobj is not None:
1344 return self.url_result(mobj.group('url'), 'EaglePlatform')
1345
1346 # Look for ClipYou (uses Eagle.Platform) embeds
1347 mobj = re.search(
1348 r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
1349 if mobj is not None:
1350 return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
1351
1352 # Look for Pladform embeds
1353 mobj = re.search(
1354 r'<iframe[^>]+src="(?P<url>https?://out\.pladform\.ru/player\?.+?)"', webpage)
1355 if mobj is not None:
1356 return self.url_result(mobj.group('url'), 'Pladform')
1357
1358 # Look for Playwire embeds
1359 mobj = re.search(
1360 r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
1361 if mobj is not None:
1362 return self.url_result(mobj.group('url'))
1363
1364 # Look for 5min embeds
1365 mobj = re.search(
1366 r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
1367 if mobj is not None:
1368 return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
1369
1370 # Look for Crooks and Liars embeds
1371 mobj = re.search(
1372 r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
1373 if mobj is not None:
1374 return self.url_result(mobj.group('url'))
1375
1376 # Look for NBC Sports VPlayer embeds
1377 nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
1378 if nbc_sports_url:
1379 return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
1380
1381 # Look for UDN embeds
1382 mobj = re.search(
1383 r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)
1384 if mobj is not None:
1385 return self.url_result(
1386 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
1387
1388 # Look for Senate ISVP iframe
1389 senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
1390 if senate_isvp_url:
1391 return self.url_result(surl, 'SenateISVP')
1392
1393 def check_video(vurl):
1394 if YoutubeIE.suitable(vurl):
1395 return True
1396 vpath = compat_urlparse.urlparse(vurl).path
1397 vext = determine_ext(vpath)
1398 return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
1399
1400 def filter_video(urls):
1401 return list(filter(check_video, urls))
1402
1403 # Start with something easy: JW Player in SWFObject
1404 found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
1405 if not found:
1406 # Look for gorilla-vid style embedding
1407 found = filter_video(re.findall(r'''(?sx)
1408 (?:
1409 jw_plugins|
1410 JWPlayerOptions|
1411 jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
1412 )
1413 .*?
1414 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
1415 if not found:
1416 # Broaden the search a little bit
1417 found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
1418 if not found:
1419 # Broaden the findall a little bit: JWPlayer JS loader
1420 found = filter_video(re.findall(
1421 r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
1422 if not found:
1423 # Flow player
1424 found = filter_video(re.findall(r'''(?xs)
1425 flowplayer\("[^"]+",\s*
1426 \{[^}]+?\}\s*,
1427 \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
1428 ["']?url["']?\s*:\s*["']([^"']+)["']
1429 ''', webpage))
1430 if not found:
1431 # Cinerama player
1432 found = re.findall(
1433 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
1434 if not found:
1435 # Try to find twitter cards info
1436 found = filter_video(re.findall(
1437 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
1438 if not found:
1439 # We look for Open Graph info:
1440 # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
1441 m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
1442 # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
1443 if m_video_type is not None:
1444 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
1445 if not found:
1446 # HTML5 video
1447 found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
1448 if not found:
1449 REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
1450 found = re.search(
1451 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
1452 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
1453 webpage)
1454 if not found:
1455 # Look also in Refresh HTTP header
1456 refresh_header = head_response.headers.get('Refresh')
1457 if refresh_header:
1458 found = re.search(REDIRECT_REGEX, refresh_header)
1459 if found:
1460 new_url = compat_urlparse.urljoin(url, found.group(1))
1461 self.report_following_redirect(new_url)
1462 return {
1463 '_type': 'url',
1464 'url': new_url,
1465 }
1466 if not found:
1467 raise UnsupportedError(url)
1468
1469 entries = []
1470 for video_url in found:
1471 video_url = compat_urlparse.urljoin(url, video_url)
1472 video_id = compat_urllib_parse.unquote(os.path.basename(video_url))
1473
1474 # Sometimes, jwplayer extraction will result in a YouTube URL
1475 if YoutubeIE.suitable(video_url):
1476 entries.append(self.url_result(video_url, 'Youtube'))
1477 continue
1478
1479 # here's a fun little line of code for you:
1480 video_id = os.path.splitext(video_id)[0]
1481
1482 if determine_ext(video_url) == 'smil':
1483 entries.append({
1484 'id': video_id,
1485 'formats': self._extract_smil_formats(video_url, video_id),
1486 'uploader': video_uploader,
1487 'title': video_title,
1488 'age_limit': age_limit,
1489 })
1490 else:
1491 entries.append({
1492 'id': video_id,
1493 'url': video_url,
1494 'uploader': video_uploader,
1495 'title': video_title,
1496 'age_limit': age_limit,
1497 })
1498
1499 if len(entries) == 1:
1500 return entries[0]
1501 else:
1502 for num, e in enumerate(entries, start=1):
1503 # 'url' results don't have a title
1504 if e.get('title') is not None:
1505 e['title'] = '%s (%d)' % (e['title'], num)
1506 return {
1507 '_type': 'playlist',
1508 'entries': entries,
1509 }
Backup clone of https://github.com/ytdl-org unavailable since RIAA fucking raid...