From fa9caede2ef0b8d916bf9c4c7be902d624afc1f0 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Rog=C3=A9rio=20Brito?= Date: Sun, 5 Aug 2012 00:46:23 -0300 Subject: [PATCH 1/1] Imported Upstream version 2012.02.27+gita171dbf --- .gitignore | 2 + Makefile | 5 +- README.md | 13 +- build_exe.py | 48 + devscripts/SizeOfImage.patch | Bin 0 -> 147 bytes devscripts/SizeOfImage_w.patch | Bin 0 -> 148 bytes devscripts/wine-py2exe.sh | 56 + youtube-dl | Bin 159848 -> 40663 bytes youtube_dl/FileDownloader.py | 689 ++++++ youtube_dl/InfoExtractors.py | 2957 ++++++++++++++++++++++ youtube_dl/PostProcessor.py | 191 ++ youtube_dl/__init__.py | 4263 +------------------------------- youtube_dl/__main__.py | 7 + youtube_dl/utils.py | 354 +++ 14 files changed, 4396 insertions(+), 4189 deletions(-) create mode 100644 build_exe.py create mode 100644 devscripts/SizeOfImage.patch create mode 100644 devscripts/SizeOfImage_w.patch create mode 100644 devscripts/wine-py2exe.sh create mode 100644 youtube_dl/FileDownloader.py create mode 100644 youtube_dl/InfoExtractors.py create mode 100644 youtube_dl/PostProcessor.py mode change 100755 => 100644 youtube_dl/__init__.py create mode 100755 youtube_dl/__main__.py create mode 100644 youtube_dl/utils.py diff --git a/.gitignore b/.gitignore index 95401e0..e51512d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.pyc *.pyo *~ +wine-py2exe/ +py2exe.log diff --git a/Makefile b/Makefile index b1a4107..e2da02e 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ update-readme: echo "$${footer}" >> README.md compile: - cp youtube_dl/__init__.py youtube-dl + zip --quiet --junk-paths youtube-dl youtube_dl/*.py + echo '#!/usr/bin/env python' > youtube-dl + cat youtube-dl.zip >> youtube-dl + rm youtube-dl.zip .PHONY: default compile update update-latest update-readme diff --git a/README.md b/README.md index 5008e5a..f04b961 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ youtube-dl [options] url [url...] ## DESCRIPTION **youtube-dl** is a small command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version -2.x (x being at least 5), and it is not platform specific. It should work in +2.x (x being at least 6), and it is not platform specific. It should work in your Unix box, in Windows or in Mac OS X. It is released to the public domain, which means you can modify it, redistribute it or use it however you like. @@ -75,6 +75,10 @@ which means you can modify it, redistribute it or use it however you like. requested --max-quality FORMAT highest quality format to download -F, --list-formats list all available formats (currently youtube only) + --write-srt write video closed captions to a .srt file + (currently youtube only) + --srt-lang LANG language of the closed captions to download + (optional) use IETF language tags like 'en' ### Authentication Options: -u, --username USERNAME account username @@ -83,10 +87,11 @@ which means you can modify it, redistribute it or use it however you like. ### Post-processing Options: --extract-audio convert video files to audio-only files (requires - ffmpeg and ffprobe) + ffmpeg or avconv and ffprobe or avprobe) --audio-format FORMAT "best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default - --audio-quality QUALITY ffmpeg audio bitrate specification, 128k by default + --audio-quality QUALITY ffmpeg/avconv audio bitrate specification, 128k by + default -k, --keep-video keeps the video file on disk after the post- processing; the video is erased by default @@ -106,7 +111,7 @@ Once the video is fully downloaded, use any video player, such as [vlc](http://w ### The links provided by youtube-dl -g are not working anymore -The URLs youtube-dl outputs require the downloader to have the correct cookies. Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. +The URLs youtube-dl outputs require the downloader to have the correct cookies. Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl. ### ERROR: no fmt_url_map or conn information found in video info diff --git a/build_exe.py b/build_exe.py new file mode 100644 index 0000000..9fa8186 --- /dev/null +++ b/build_exe.py @@ -0,0 +1,48 @@ +from distutils.core import setup +import py2exe +import sys, os + +"""This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package""" + +# If run without args, build executables +if len(sys.argv) == 1: + sys.argv.append("py2exe") + +# os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) # conflict with wine-py2exe.sh +sys.path.append('./youtube_dl') + +options = { + "bundle_files": 1, + "compressed": 1, + "optimize": 2, + "dist_dir": '.', + "dll_excludes": ['w9xpopen.exe'] +} + +console = [{ + "script":"./youtube_dl/__main__.py", + "dest_base": "youtube-dl", +}] + +init_file = open('./youtube_dl/__init__.py') +for line in init_file.readlines(): + if line.startswith('__version__'): + version = line[11:].strip(" ='\n") + break +else: + version = '' + +setup(name='youtube-dl', + version=version, + description='Small command-line program to download videos from YouTube.com and other video sites', + url='https://github.com/rg3/youtube-dl', + packages=['youtube_dl'], + + console = console, + options = {"py2exe": options}, + zipfile = None, +) + +import shutil +shutil.rmtree("build") + diff --git a/devscripts/SizeOfImage.patch b/devscripts/SizeOfImage.patch new file mode 100644 index 0000000000000000000000000000000000000000..d5845af4641a3a4028d70fe47ece829bcbdad4e2 GIT binary patch literal 147 zcmZa{;KY> VM{?ptt`-3kK&n7`AUp;j008^TDJB2_ literal 0 HcmV?d00001 diff --git a/devscripts/wine-py2exe.sh b/devscripts/wine-py2exe.sh new file mode 100644 index 0000000..dc2d650 --- /dev/null +++ b/devscripts/wine-py2exe.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Run with as parameter a setup.py that works in the current directory +# e.g. no os.chdir() +# It will run twice, the first time will crash + +set -e + +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" + +if [ ! -d wine-py2exe ]; then + + sudo apt-get install wine1.3 axel bsdiff + + mkdir wine-py2exe + cd wine-py2exe + export WINEPREFIX=`pwd` + + axel -a "http://www.python.org/ftp/python/2.7/python-2.7.msi" + axel -a "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe" + #axel -a "http://winetricks.org/winetricks" + + # http://appdb.winehq.org/objectManager.php?sClass=version&iId=21957 + echo "Follow python setup on screen" + wine msiexec /i python-2.7.msi + + echo "Follow py2exe setup on screen" + wine py2exe-0.6.9.win32-py2.7.exe + + #echo "Follow Microsoft Visual C++ 2008 Redistributable Package setup on screen" + #bash winetricks vcrun2008 + + rm py2exe-0.6.9.win32-py2.7.exe + rm python-2.7.msi + #rm winetricks + + # http://bugs.winehq.org/show_bug.cgi?id=3591 + + mv drive_c/Python27/Lib/site-packages/py2exe/run.exe drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup + bspatch drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run.exe "$SCRIPT_DIR/SizeOfImage.patch" + mv drive_c/Python27/Lib/site-packages/py2exe/run_w.exe drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup + bspatch drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run_w.exe "$SCRIPT_DIR/SizeOfImage_w.patch" + + cd - + +else + + export WINEPREFIX="$( cd wine-py2exe && pwd )" + +fi + +wine "C:\\Python27\\python.exe" "$1" py2exe > "py2exe.log" 2>&1 || true +echo '# Copying python27.dll' >> "py2exe.log" +cp "$WINEPREFIX/drive_c/windows/system32/python27.dll" build/bdist.win32/winexe/bundle-2.7/ +wine "C:\\Python27\\python.exe" "$1" py2exe >> "py2exe.log" 2>&1 + diff --git a/youtube-dl b/youtube-dl index 789724041be58dc3494057bc8e132efe0e090aa4..f2268af227b0eee8548aa6d106861cc508879931 100755 GIT binary patch literal 40663 zcmV)4K+3-(Aun}vaxY?OZZBnSb|7$hbZBpG3Q$V}1QY-O00;mMbnZaRs4QdA9RL8- zUjP6R02}~DX>4UgZ+C8NZ((F*axQRrRa6N81KVZ+P>&cFP<40<009K(0{{R7=mP)% ztvqR$+s2Wf$zRcyb{NNKAEaz9w*uQST*%( z`bT^+946Z$FKe}_>mp4S=Fe?ZZ_Mw!GQUb~epUJMO4s(UW>J*+QdbtTy0bs)WNU{t zWeStLfLT|7Hq4@}RwpMa$m-yDI1r#^eOKseI2f+Vd<&!|X{F@Vli_f9>uyG<%?hSvK|AHM~@!iJLO&~8W9eM19}m=7d)y`QK?nE(JIe$1&geTJgbrg zz~(Cp4tztH;UHSpO_ZwZB-XiFA;u_+)lHey2$H~pa}_7mm4Y=Q(x!r$HW4EJt5xDt$%_#2UtL1=w*91kT`fhdgxkGE%FLA{BAht=r;rRY!=uh z%cU+rC@Rfs6m1cK{%Mc^D=l$eWjXP%it46}YS=EcuSmpTeX^=98)C&xQg2}2t(&dR z>IyXpR=3n3LzoEm&l2g3$~%~7ILHh93*-aibrch8D?lrcar;KsQIfI{Hf7$dH?$Zc z5!qeWWwK~$5G?7T?kY2o4EvO2)?sFpRTWU6!sP~*v~LpR1u8XC z8{M8^~c=--a7-1CdNvpmudq6BdTQEq_A6#ySy>l4v;X!ys+!4(9MfA)kG#aX3*wiyE;$J$tVecT0|vo#WcE0VXs$pR5DNA z$rG>+ddnLEUa6?|pmk>U?hZ7(Vpx#ta?{!omeYBb-YI%-cLuW!4YfbyjGlPQMRx5D zO!IXOK?|6{QU?B*ue+NXUK)ddaf(2dmuN9(zfA?cS@DSmB91xSUbdGy)*q4H# z61IRTGH=od69p6`{AfUm63w;88oi*ndx(Ao9j-c2KSTUC9L!-4!XFox@UtlFPcv!` zI}i<(gU_1n-07Nq3@g?;n-kOJAw({ErRMV_OX~T2gg$v9{vOi+r02U*quGkO9$mr_ z3PEs15;?M44RIlZ0pbfi^5ubBggoe3%FMp(uExD-FX#f&rI5G7M9FX1ESDF&o^Y+} zQQ&Q{V4}WRU*x&|)(uG_0WeJcXQSgvnnNS5WmA^O6``4)?jiViIz@WHN&>Gjt(>x>;*o-g&I9&VEq z9G3=G$Gw;_Qm|q)gu!wF6{=4i8HEwH5V2z*qdfwLAS#sf2u97f;Dja$f5GIS8Yh79 z8q7VuoT#s6qQ3@Iy;NU+-M@A#tJ#bA82>{;>Y3w!ASmc*iNbBoJ-l^0>(e-U^n6A- zh1#}wR<%|juRz@(;=*5TqZzi0M`vdN1^aYu0S>rPNXO^sI{lSq z`UZIhtKCMoj2RHW2xqYULy`zVM7Z|>ZseezIH-yG;`!HKd_MbP416$wDC90!n)pKf z^zR~9j|9s&k4>afGTYdxp`pni&Ik$MrII0sP#NHFl&rTOILSP#M2}g2dpQgiztX-@ zuS??q-{C9pfAENEQ&~)T8I~((IhX`MJPTO>#HKlA_5@bzHAtKtA;o&&F@Xc64(YE^ z8H~P7#{V_?7(X3-d;Bqk{~v!l9$grdeEIG8-vW^?P{RwihCl3IbQk@iOLU8OE(42!m*skkB|MM<@cqrDl-O(?`;8JdMIU4Wq77S@TlDVHK zHv#E1vKZ3!IR;sCZIXG3Hxg}F{VIl_JL}YYB+!tW62|2cLt(t$4K%{2C;()XkfBO+ z2+`);l^1A6pCCv&>baD8EI~!lB1pXhpmzplKRwG$lw>V}AkZ-c0Zw>a6o&|^a|vmL z4vGB1d0r*OT<7Tt2P{T0{hZ&W{FqDOo&`HVE&Epa}xe zGTQPkBI3}AklCP#6c0QnY_kZZ1o2lBe9YiIpt-y-6k{|okZAwJ()v`nkwb)lIfF%D z$li(WLR7DWeCC2foZFGd zKSG^h3_2>PP*_4Xl5xd<1g0_IT%8!T8-<3HG>jO}9*|YqRGX16a-yUy@)K?D&qQLWq>h|8B1vXOrfHw{_~~g|iP(zp(9UrN z9SZPCMu^jo@V+&zmvH{app>>T)I+q%})@W(bF=Nei)vYY! z5z7L$A!_t0SvRE=k7+|=nAk3|p}7IYnQ&QI4pg?$9hJs!&`s%EEafAWIT{kwG9pvP zV~Ki&UU8zL6)ejyjcO}OUbi@R+2=4HBD4OT*y}8XQK;-64r@jM|wpO9ThE&;=uzhxx(x_3$*o5 zSRJdFjSFRv=pKsEr;7sqw%x(Z;DpLhA^vB?Tb1I|cHZDctLFVTE-|q10miDpQurnIGd(1h~0Rwx#wtDMQEmKZiIA3q~pnnEAtr` zdk(_|9$Rp?TDv0~t|MBtLkU=9!2Kqe2KekFIo}aj!2(Ma`WDN#BQ|Z{3;f7ZIwoI$ z%5DZE6~5d*!1peefZ@&)2u7NKYMOy?3$|=NKghCV<@f+5r@|uG$YAN6_u_a8KOD^E zE+N)P-odu>X3o=iZeS*q^0a6H_+ts7d`RT94@)pkM+d+~nWe)(9|;6Jowu#&X`_%4 zRHHX<$(Bsi+xPSXhnA&hzQp!7!RguA+p}ZfvN`Q7Mb7|jF!s36&~9wAf1=U|B6d?& zo^N7nY2lwvz$F#w5xTF;f2;!7?Lt5gv9>60 zd|O!`pwyM{|NnIQ#{o0HUxSYjepzWp+yIX{^R}I zpDnpUMw?z3S!0stgum-}h+Ge_>tJ?iGeN|qdhLcW6&8J)4s!rz(t9p6LpbHtm-#R9 z$NZD3j;;!1DD?tR!g<&a3ecW}XJ>66OK){Y6&ft~FNhNUdwTv#QCXec=BF*fDClc+{#U9&4e7$lgDJPv<=BAv-}PP~ADYI}T;;Z5D86HKYlM={z&$)S0NMm{CS z1K6k{N7Oznzn{b(D9kT0Ty{f4pd9cbC z?bmtFhPBqjB+X=1R!y4Txs=PKkjC2_THaB$i_Rg!bvRPIsN)0Y=%?g>`{R!O`OZxP z+o4%Itm24e4j5S9RgxvurjG!t!3<)B z@F@oBpv^V-mpLT)kgmr8=S!9DA8ScZGI)fIYVI_*wSYI~J(nfQn5s?O?FGDCQ+^@^ zW^B-imK#5a>pLud>hjhCI0qTI4PHI>ImbphL&Xm|)0B&PwULfXGWu*vCN!STzn1mL zA>0AE2tY*3O>haYrnp44b1&o?_YI4hI_HJA#s){TFKKJ^ES=D}TwqUy2FA9g>Q9)= zX8T)|Hd>T(!+7#$8Vh$Q& zZSIcvlu2bN_fJ+U*Q)B;f87>W+;B1AV&;j%cM))<67w|I0n}HCaBuA<8oHwz9%fQk zS449!CrbVgUcUL^=eK94-@ST&+Dal@8}JbZI*j(I3LeSNIN=!xuPV?Pz(9ia=)eJd zUTHg&dW$RWb!ScfKwOktkoV?6E9zec(W+yar@an;Znu!GlE_&Ooeu0%5;@DkGd*yf z#68<7%NT)##of+PR4CfwX-Oq&f^3X32TD0;|RTs4gMKt90z{OFaG@wT6zR072!)pQ2miN2)V4 zndMzjcK;+_8asj^U^!VGlDb@b62Ums57g^WQvxKp*_~CDEu{n?9?c z-JQUcc~!yRkrHYexkrHn=wILg?IP~;pmtOY<*;Cn2eHypO^2v^NHj3LIc~dF+E>`} z;;9&IqA9J#N!eGOY}FpAA1qU!ov0w#DGy6MIfiUeJw~v^1^SGXx_Yn%wvDbd%tydp zyd#y&iIUppCzLdo+%ySDH9C$vvdSiI10>_|4wH5m+pV8=A-7cK1$dW^1Oyb6>1M%^nxn;zl@rYSHflgu=mO@(fI9lq9aPISujCYV zuVjwh5_hEXU6j$$9Ch#Rlo$PqV7o#-I3yQau@%oH91bnr@;y%14tlg`EuIt06mI4BY>3PE||6s1oVvS6Eg>ngAn0So`xawqTIPRd|BZt7%?cm8+Tt?S3s| zz`$vARNt5k_x(WignDra9)0+Ve=ozE2ZXcV@aTps-Fs&b2_M$5NcG>e7W2 zEeB@E`!3IFu>R+H{(b|dK}dYS{7W7P1msvgDFK(hEg+-;$8l6iGcSsjD%1FE%)Q%X z85%&EC3Gd1E}ce)Scp5aCDrnpH1C(F*R2WeY*(oLOT{?~JL)iRxzb>DjH?Ww19R+1 zwXIC{C_&C5=z1BLDo_&d)QGwQ%uq<2p9?>M#!d^D-@o%77=XKm`*-%VUr|C0h78c7 znKH9*0NW%rRkO)d!`7iaN<~Awq^(^WCSCxf)jw80zBlsIT;xuu#5s@~;U`F)bR!dS zY?RHFapZwO!lR2zA>An_t5RQGd+YYSSUW>-|+LzW8PFp`__m~Dw=nk-S6UA~` zgN*zW282eaz%3fPfMD~=^yT`3W#%99K3Q&o+w+qNr?yhIhw`uxvQs zGC3-}@pE6xVJd6e)nhC^FJbpm+|XSxaBiaTLZ>e0W+(XppMTo!4#GiSw?((C=9-)Q z6LZiRY7M36$B@AvaqzNq_nDG#3Bk+gA~2xf>4Pb-Ya)&4CxqQLc*XiUYtj^hF^I<} z(#m||;d_7n`t67FgDLod?ECEer+02M#JXlb1re-zWJr93r?hEoI)Xt^t-}U3f>IC> zL=EYYK@d8|lycGlp$y*?S1%~98gc@r(R*f^+gRyL5Iguv7>QJI@mx-wrHPe4$w9;I zU1)VKA2py$x8OZ2v$6gJwk`z%>5jmw?_a(7E4Gs^qAFR=qjF7`EZfrgEx=^$;sRuJ z5lnvxe1*Eayl{W`^(`6_O!2}b>0mI;;isooct|4id6T<&G|hd5|9u|7G=0tCa#Or@ zDYGxMgt&460rH+=h_{EG08%Bcg*PL!a6(brE z65NE|kNpVz4WaeyavLpdz|!eut1Pb<&HB>m%rf6@QS@m7mVonCSQl0R(Z?6tdwKk zi86?I)3%u`rW}62^4{Zjh>%YhIBL90jBCZqDA=Y#P2m5lzWzG=<536(2EQakr|9US zxXU)@80^g)Hl)gCQrR6NUS&JoffMv*?^`BhrRjf~unWwX^d)9;S8qYjyFF9qWbzwKuvbz^8 zy(yWI!9R+*I|$d<=SRf7&pW9?UkZa1pxr&$ITeBzo?Z6dvq5z4Y%9F)%IVWuQauEn zsz7XeENhPuZl&z^HM`mh{Cy4Kj@kG0Ang?zE%23V`-=OWH7^u66H_M!uQ>(EI~E?0*M;=O(>4 zoo4wI4G4DdU|k(N3bH5o9~%Y8*V3?QL`oEA{A5InL0Tpg8`K$yB1I2>E@_`Fxm+J zPjy~lyffJ~}TkqQ4seL0&+I}$&6d7{3W&DfmThoBcP?e|pva(jcvtwjq82%a|V{^UgxN42M4iDF3`YtETu35Fm`&^xEUc@YsWVijbymO++rWhM!f6H*M29j!7f- z!*WTpa^WL>YOe|*PId_2K&rk>%$NMIr<8g?Bgg~RYu*nuCx^e-t)9ekXOKU(WGRz| z)D~;a%|l0NoUG{kk12oH5n@~vv0J}GC?7LLe&f$gRK_YUlWT3|^~phu9rU0$l`Xp; zt9Dgi)J!xQKTqnq04M53%SX>zcQ5g1rf&f|?ah{k9lVDe>&?tU^A*OS-ne&Lh1unyIGQ498IEM{a(CVkztlmLm^`W9*Vcr%DJkM%-JQarTm1x`^_ZQ$Ejkio^(~p=+ry&u|&BPPX@=_UoyCNKRMO9HQBwg-45mV zeZ;J#0QSmS@FBymO=s#TIB?I)LYU|)EMT$^kTl$9JNBvUYiYa(Zvocxj;K|nQdQ5< zg55f<%cC7b(6yce{)H@bi~i3}ETDHl+;`;Pk#yTNSyJwgOepP_-X5~Pe3Mx|R_wl! zu#;@MZ-s&`QZ=1QY-O00;oN ze*ZuRKnXe&OaK58e*pjy02}~GZf0*qcyw}MV{~tFb1rasRa6N818(CNP;}!LP<40< z009K(0{{R7=mP)%+&pV{+qROQrC)){*Cv_BlwVC#oj5vf;-o(7JdWeG+m%%wT7+yi z6shuITfNDD-CSc@lCJD(%xkl_jr86X7l_yiQ5}& z`_VQVCqWor^;wZmwx424yB$tvNt(03&v_nBxqAQc{LSfmKh1cmTCVea7KNAUHRgFb zRxe4Wp3)LRmW)60T)pYAi!{;ziMKuV%BOx9xkVmECCs~NERkEl4oLzGEHi@}|+I-%?FX?hJBN0U;u@FR=hErH>HY-w)DyQ{M zn2$5ttvSfIxa8YGgpFW&ZaD5&WcShsV*-jbK?Re{XWUOgWhO;T>(FIcF}`lMXs96DbHrGfnl?eRL0N?^uj73PZEn@YYY_g09Q1> zE~b|;n5=&5RS`ub9^^>rmg)xE;a}l;z$P1~A&I5mdKdHUD2&x|YJDix55=Xj1GMTO zEaWP?PKqc1YPSPE!O6(3hnv<5<1i2XDEyP#4sOJDOSDUy0gY1*n-muZgej-{%=`K1 z^yH-{ti%<+n*p8g)8x{JO+(v%u8{gsmf*_!K~QbaCoPO+|<=U;*;9KpZYR<6{nMg}jt{rcHE_ zf>sORxpgu`YsPbp9=m#UBvDxr<{$$uiGr=52&r(>ila1Bo);=vYG2z*C<8a2c8R4q3Pt}S_xE!+Ey=0dFukxSLX~*)3Q&VNaC=%(u70yJ`}!mWquWg^5obyoSFu2rrhQvEoop~I2c{I=PPS+$WrS!+aC1M zdCUg=Ue6i88_l_JFM011yj|QrzHps^1OyG+aWYPlk0Eyk7YMiK*n@9wLVoMvndG6a ze45Vb1)sA|ewt&OYJlN=@$cpnpI-9d6L)&fC((3EJ8sbTX34CW5jhv`>>8fL^Wf9Q zHv@#=KHhQ$H-`>&40pD_Kid8ufBR2wd-NE(?On%x>40l%k5Z(o<}BB#LT?d`3r ziKaKWjt+UeH3PQ@DD>cMck3!T{OLp4bcIb|$2^$j!EqNH?h^2KuTS1e=mb1#fS!d{ zaWNajur!A~36~HegrnET`uv9JH#`j|a}N$E0)c1#BABTfmF2O8@AI_f1z>5TVE*9)2ZC+tG(?gN0UnboD&@9=dxJCLtWjaR_0 zzwVoZ_h~R80>m4*sTpil5HNYav6ZuradL~t6Ir?25L^~G&fmQIxyz1zK7mC6hX_K! z8+55#$Q!`B<(E?dANV%kzpcGL(ceFpOa~DCnM~8fy@Q>*gPk2Qm!=K|1IhNj`AU55 z9vs~5e)mKd%>I*|yC(-bx^TYR+qv7_dn()Z4;F^)6Y_(nJ9kgNHRQOv18uwC@9pT) z+~4_b52*ZJq7@=B;0dW1@PuGYN`!C>9G_bd2_AcIkKW)$DXN_!tbWQd-t;0rzN#H5 z6o>jbgenlglCzvM99E!{*?e%hdE$Y4$$4xS)=**^F*pUu!Bw*@r-nSa3gZ>nVGQi( zhv7X9M zke&In{_^wPE)R>Tz^FT{!JecNJ1Q{MFTqy}b za_t5DHNw+;Ix|(KikJcnin!tai_@^kMC;l4oA>BP=irM*52Kr=mQVwpbCg19V7zgZ zWIXW3J_W6rcQ=hZgp?}5Vnap=NFo?I(2WBBsg;9J@eGB~5#0I!&UHJro?$D4pfAl2 zJNED*xNyN0>)0$PQoX?$JO?-J89YAg;gsi=j;f15-~4oXesT(a8wOh&?EKx!cOs{X z11YkE_z#^n$`imC3;`grvc4h5_1vFwG3jy|Wexw9Ac%OBm%o&B)GHU*TP{bc9SlO3lk zfb^d1Y(CkMFWS@YqurgI9S!9&pwL@^#1{z=9?RbvFewo{>|>t68^=Vutpv5k5b)}P zq`1MJ z!FQS~O{Vmi@}Z^j$~RhI>SxxG+P12*Z%Ng;L)B1?h+WRu%r5f&MUKCQo7o7qyM5P{ zVSn9^cCUDjz1AeUu^e+$<#?8((5Ead#q!7V2F^RfR)HFhf?AJNxF0)gZDVTpnd@* zNx)J7VDD3AO;0lsV;VaPAcocS5v^9mXv0|5OJjL-FHJ-}&_oG3VI0H_4q6F3wGYLO zlVHLA$$wgR3rM!uxbZZmAOFlj1iXgrE?cu3(W`sL{C?aABMeKOs6 z+zYAo6%}9`V4x!eR4MWX1_NavKpXy#f@f$`i(*phf~%DGPb==>B+q{{EOoiEKaJFS@)Yne_w%J`I|qTIAFyAYDG@ z+M;>PDIGrJDH<`n3`2!0HLKZRY+&#Mx&@CEU6BEmfic91<04JDG^-df(irrED2#*w zm~^rF@X%8&!lx9m8uVBQ|0Gtu<(}=AIxl<59ojFoUi5rkXZ2NPsw)TJ>hgG;1l+Fp za+WFw1(qaYF*@a05s`PZT3b-c;I4=(AaH@~LFha~bi?1Cj~+2(@vu`0t^zZIP;H2XyrMZ-IWjHXhmL|>!c$q*bhPr|X*S0kC8 zY)~Ra#Znbo$P^G;*!mD%`f@9ilA{Lea(s67?o3}5EopK8(yU7UZ>XpabT7OzFOWr5 z06yG?`8Dj9AWXTuD~-aB9I-RX{gyXG1sxY^u^AoG5WiqDaGG5HE+P-P>(pUZriBGn z!UW>zzO?-JWS-IeRO=r9Q6vx^;7s=lL^rc{9mkC@I+@v^b57gfSB9`b$)!Q#Jr`>g zfrfovy?kMO15RRqxYVLac-1ynF)2`?F|sFVf6a32F>8nI{&X<5hzZ|5w}1py;dJPR-f2l}7JqG0lRN4zse-p-JsiE4!VLnY(fy z4LUYS{oKcJwERVc=KBL>jp|!<9kMMd8;u5^(!1sXX5b9A@Pqy6f z8@!9KtdXWqgiTPnRBa(L&}bWMDOx~}L6c`0;F|bg)quxt;R zD!^Nft3#}G zssqwsO2`E_OcCRK8cl_z#px;#F2z%O}^2%#tLyo^b|%s+>(S|5w?m)K*z{ zg@DDfoX%=Gy~|ODHQ&UWMTNd`hl)hhDaixf2Pl2V=f9JFtqQY|a#5eTnrs0e zKI8XC)P|Wwk-F6)&?}72f{_gxpz4SGi8g_ss?pxi==NDd91)f zEx_(CHUYYE#QoU2&Zm*R&|P|YjWbE}Tf~*PTTqIa#-n~kat4R2QxT*mjd$NoQUAaRn_1eO)rXIWKT9sj@DKxO% z=c@f}PL<3NpWkQSYR*(CxMoi(lL=MDj&TJj5?=Q3x6bI1)4?RzV;YOvNLPTj7g)E3 zv51mLGgK_VwC!lPJL<9s4IDWq5!DasLrY(kd<4B`75{=+h(2LzU0tmb?7f;-DOh^|oF;3iD^7w)+qq<`GEU;J4a+wj#f4GBQ+eBdye-X^Z0R@_+me z)s_|b`RiCLJ$%#+xX|nS$_NAr&0&rZprnXCun>x;s@z9>TgQi@7*O+_ zUV^R$oQdy$RNbd06_N9jm&~g+wHT%DHrvuS9Vc-;sHA{Nr;El=Y)BX68Kz8a3rk2W zXCd)yR+eb~C~&VV>Ci?Ry3I(O@=DagDI(z=c#~<4)fn8LdQ*SaL~_JqL!9@_2HXAo zknO50EILd+_GhJ&zZwL|gS?q3nNMIa(siXrH|TLLT517+n&RRlvZ8%c$+0r4YbEGg0nB9&0SZ4^$!H8o0% zE%gduG}S5#K=&6HuR7bbv1)#K!ZAE*mSK@O6oWS{Iv}~4oiSSgB5Xi8j1+lc&~Tx} z1xJM6**bWqKQfalD}V~q1f3zA9a{PK9jn!e2nfLGomyDXhmqtSw?`d{!wwJG&bpZ$ zI9o-^M$zEw#Vcat)bI9IIxI+XwV34>O^P0~jAnuUMP!;mlvfyd^{AUk@J}w`=(P0I zrD2@vSj6@b(i}E)g2`eV><9E7%7T4q!fJSLlk`D|8`o@m_dar#O%X&A0ryb}dShGL zm`b<6eU|pUiPlL)!$jJX2^F)|a3j{EC}O%Umx{l^J_vtotomK{3um)%{84^*nr31* zGVn#SYX~r52XQQ%%+LXf$N1eK0eVGowLhIv;TIepyS=^Me(%6SHIUvCmnL-262GAn zKUpl*xQodULJmV*li zrzHMYCcUe0x}2xq;jOd5D=|rJFbhi^>9&>k_G%s1G97rp<5paLrWzl@m0o+5Hm;{x ztDEd0N@F5EyFxoi21fZ8bVZCeDy7Sy1VqHswW+9ttDKdxgswcg8cXX;bfU_xEr=x2 zTz>}6dSJ^+fe@=n%6rhd3#v*|L?wYq`?9ez9Sal8{OrBsDL;f2FIu)p^o!(YUr&3= z9$8mYlfn`-($(y@3;~t)8228f{#b3r7MyZLR+-=I*He>ADA1BX`BK zzR2Dx*yz^-4IJ=U`{+_GclpKO8=m`Pf3nUegU7}vgHt(i`8R{|i$V3hAOxlOg`k>u zQC(~I?BQ>{(PO9l?LNX!&|aQ=cyW4k^5*!gR(qhM?oIq@7|lK5kM?lhtCQ37<1_F5 zyAS90PVAAtgP#o!!gyBX=a`QYaJIk%fMtjxp1#J!>LC^fS>|Ux3YhxNXN`}a4GFtT zC)6t7gD``&g2NKb=4l8?QG3(Ey&%Cy>WbwL|4Mqo?gM+-ZwG*Cuu zn0~)JDN_17KW6#Ddbx7LFAy%tX4YF)O=lRm`f+4jVSUvVcWI+YSK2zYJ%oO%A`9Fp z+V*HJjBKnKX}nXn2UBNj%7f4cH07Tq=(076@6@b?EVjK+an}$*soBHr{r;$323^GQ zRp9>@pW|*S15HA>gnTjlkjeqJOxha_LKRADbeKs9NkM(9*C$(>*}(H)u;;0(9R;r| zxiAV28#s`~_Y+KI-1uqMVPK02yoE$|I_t3Wuaw3RnW$O|$vGebbH#e?-Qs#;GKE0A z^mD%|UlCKk1XT@^>nKro`l76q1lH|*Ur8PMv3DQm~@N0l(;HMDVO9+LjuF2OO}A0$W{_WlF1| zw5r)0WX=!s{zdlana@Ox(Bpyg?91@*;PSWyzd@!7hQe)KBNb$3f|ezL9+(5Fr+_dG zBRZDlejbwgsWLwrU?a;y7k|+Fc~mA#UV_=5PZK=B9-J@H<}S&WgwDd>SduxDpqhK( z4*l&vNB`XG?tc~~HkdW>Fm2+=^hqsGp-ukssfsTjoonoTr2+Z^g%6F<|NloOi^mCN zq`#D}FNu=_T>pRD`_{I$k!;=1;a6xo2V{e>A(`%(VdC)6xpXo+7fwQEdd7D95w+W40k$9k=Te& zPW*46|CT|2RDnOl?VuUhq<2o$ znk9}Fj_y^VJ3S~A+P!CWql)%?QuJ+fW7!jui3Q2|N0bU^Ar*$mV*p>vGmg=Q;G>hY zM;Eq(0LfTMF0Bk^)z9U~xY3Me}hckcN(*kHU$}m(pV*f7u5hNd}^d4F*llgU6}a`8Qzsdjgg} zl9lwzw4GTK>&5qQl%pkCS~YkD*@!(em{s;KyD>eReZ2f-eED&5*_~7_yTN5IybP1e zlgVW?y&MfL$Abz(PwZA}zZZ9-9x7>CsHLI1I}?AJjrxp=A$6nO^T*r&4@mQ^B$|eM*5 zL5HXDG(Mi;G+p^9G*0s_ZIOtY^quX$wkdLfQ2?7J);621eK^txlN(;-wra4iXZ7nw z#;@UcV|3?^o^En2{C2pxbyfPelgVwKO#DZKX!H?vsg07@cTc({?{pHLG)f?r_Aw?{ zgB-`VZcqpM*$0U$Lw)%u#}}U10vKDxoj(T5(~CZE=D2k|>0Lzx{|Zjy_=;-L3(RX^ zf%|5JG!o?)X=^axO8$qwF|EevC1JZau0Mu80>MuCPz|^W;%rSLY zRZk6vu>=cZGn!I_RcGIh8Mk}PxGCEkwzXiR@0^iE?IsN?LG7jL4o&7Nu~FCRt~fKY7iS%_S*~4nsb0|#e1T3;y+wVZmu@e?%=WD!@H$mLlmR_d zaXx{-1X(!^;ar@|CLu)~r6V=$^@)lbPr=r!gvHN8%ASPb5DTXplVQw3qan+%>cxl> z&LaumM$zLZ@MDC*&b>eF?AEwT1LE#`xAW=K_fhZPqklg5bujtg!@qp`bnx<>?|Juy z^jI~C!lc5UIDuA;h}i|oIF6Z1|FO(&)<;#si&&D_zFsWp`KeaUey8*|7j=S>4TY*^OHu9 zJlvS{8X1teG5y#W%?5*wpr;9^8)v)Kvj^1;m|mkPbZYnE&hDT8`sc68w))!8onB=O z>hwbV>2*O6gGcX=-aR{b?HwE*HYA8ay-vH+^AE3H9UdILXq4*v<+@y5wf4nK_By0c z{^U}mA2xNq^AyA|y=>nw$4+LIWm??o7uTdgy#pH4Rmo(Q;*yzH>T*iie?-IZDoR<% zwMZRHMbFiwroK_^9(BpBt&mD}7^b>ZnnX#RT%tGwn``XXB@%qzQ;G`sIy9Wnkvpr~ z)NT8w)tm>zu1AZ)M1>d;Rrx$6-?m;W7~nY??;MByD4ohkJ(PIES@+bM>F|t05QxqN zp`}1tReP`IJtro>^DwVX6=AK9N3uUi`&MM{hW~)-!Ut~~gk&C$6=aA&~^4!z&S4rw5 z>W_jc9#2_~JA{Y9aE!~C){e{yU();SrmwS5Y!02pomjv2+w&X5lM3fSmzpAUtCcQuE0_q(yr}LmRqt{I-o6ibXw{Fw$K?cgiR&t7M!r;EntLXx;+V#0yqu_5~m4l z&ONWs<$VRm9vzh7Ip8Y9aK}OSqX$DGT`U+TT4D3gEKz)RY>Hsgm7jTX-j!@>2ihV_}S%|aH)<5p0M4DQpm0mnM zc#5k}l?Z>nlV-hube%)2D8QP8k8RtwZQHhO+qP}n<~_D;;~v|1cQZ4|OJ>olUUt%{ z?!W4*ST_JBh13}F!f9^0m~5BZ@2d=G!TB4B7-4ppg5vpKOsgGpWcWwS-hf~+q$8<6 zM9X&}yqp--Q65TnQ?tVc>BAMKzm;C(1WSq$>b-_QNqTb0EQ;sO9#+yLA|i7?4(9C_ zyxX1uzdXbKKQ)J+4tNLxU(y%2r;0dD;M{&AOO&N#@}bdh8ob6>?r);&lh#RPXtG0w zM2cqTzdSiu_y;6Tai4LAAk^u{Zf4#HaUV;{EH?`B4!|^#oY5Zzrz!hn>b%WxtXQ!v zovN3-;BKRa*MAi4Hvh04p-l47t23RK1(%n!80HWHx{w#;FibFX zHk*8^)LMF`1P3hBtVWW+er#^MNGhZ9v_r&s;MLBlyzf0eLI!z_@(dK=s?LY1Bs=m* z*>{v*Tr{7yL3vs*{VhtSG9RoL*h^VKVaPxC6}EN7mSNeUT-^26oaTPcq!?ZjK%TMXGVpD_)e?fO+Hl5gJ}$T>VA=hBeBV0KH(2as<jdv&@X z-{p`?iYOzgilJ6<}QkizPnUN0l6mOku>q0dWQ3dw@7x!n*-y$5pxhbG?fzY4{-jzv47s%EzMlae5E*F>d-ZlA|w;y z(iyjBC|FwRcNF)}B*n=xmsu>pM%lprsnfu<*^;qQrpFs3Op5ul`oJFBIc43GvRlGH zp6_|tAqB3^i2i*VP=+&~(fGqFyPV3YA8Mj+OBo;!fK5_kJ$`;dSJs4~)mtmAv6TAV zYesF^Yj0oZzcz3nEh#x_3Mo^}CnnT_@hm^skgiap-stiMU+LpGv{R&7;V1s~T1ht+E3Zf-uNhAoxFYxjI@2@ktM2zMsBGDKdy z(ZGQcJ6X*Kd>}4F@O=)$<~#0#BPRkZ5-J4$w>(Q6XBN?bA#@2~VCG=uth(DIWK$w6?p$BUz{WDN@)f79MBTo5A@?s^JVl0R^ zNh$|q_nv;He;&`uJfVQJ?Yn6*LOmkx#a+y*tEzE(Cg3Vi7h)7frwYgKRs((G;UrI- z+l?>)W0*m+V>L=CKDI}kE9C@AUr^pSub#nKVsu@|b+tjz* z$-IX5md}u0E0Tv62*#TmGyrCiI|6KKr6v2`yV&$n!fv7_F-c7dk3dKk*l!9wPX5W@ zFddGAUH|g@S_2#vWdqPZsG_fY@Nlt+q31QcIh`wo){- z7O7WJDeyI$D7ck^3RY_k>AuUTuzs)4dsVRh>|fqsWWtVzDv@>8g9T_>Ny#IL zIs=+~2JgF$p|owGpH#&XZkgTUeMLapJJJVqeDP*p>W`A-EH$un_rGwoCHW^9HTsYg zEa^R#CF#Bf2zITn1D#t+5SzPuT?FKHTqx>2oeI+tsv?(Rr_Iw2 z%w3sK63T$sS;ARiF}8_j&o)@tbPU(hABdKtNu3G_uw^q~O1oFsmQ4%it`Jrm@nt(F z8(q}_OC@vK;*%aO3OQbj&WMj63_$ysZ@NF-&=q(zL^WLx3whDb+7qYPDxB1!l#Jyx zqpc}7*fqLo_ZQez-%|g)wS0X*3f9l&tfqg@jI^pRj8rQ#FR+VTi8l=b6mE->1Fi}e zvXQQ&-6wXs0bKeEx4T1~W;B~g=2@Ey91nIt20i0CxTF2hSzJBGr=pBSdz?DYb&8Bpw?`XLek9@J$4CiX=z@ZJQfyIb-5CozSU( zPBn4Ayqfyu=u9|H-?o?+ZBcpohJ2p2)SI!?@y^D95GCrGO z3Q#tQnJXKOJZe-*yN=;oK6HaS~9Vsal08gU@P7;FL($o|rW!Znvx%8>VEGd{rGR}k%Q#Cnr zm~sOI0_O+N>S96>sZfOhe(xVKNC412c`%CVzfzr)y*3c=uz^F%Hiwh4t!~bH8Y>4E z2WL^{zSaN19$0Ya{W8oxidv|{ zrc5c2$GuoWs(^i5Sk!~fS8SsZQf^{3JS=oAItVa)UX?6M$+d0y0XP_h9ryV~EzGSt zx7+7h!D5Gu1~dG7J|huXns%kkLc-1AqxrbIF#Mc!uFkQPn1AG}h$055JyG~t1p+ig zK)_go#hIi&e(7pNMzkHHQA>47PRnN5G|c-C~XH!w!DPKm}M4A7R(SuoWbw`OLH-%MBPJ~i#%A?fm@|HAS@dVTQ3uDKqToCGZT z-U__y5Tf5Pe5AI|+qzVxoeWO}&*|lqhv?0hR-uQoVk|NsRvj#$=-Mn|X4Bzo5E9Q) zJ=ru=&V?O*ish+R!JY6);yG^h;Bc1LOc=uOH_BL)8f_IfW0&BKbc)7deoEjMIuKqw zP8;6gb(VKRg2}Q}Tu~ZcAC0@Mk5wlwAn(Gayp0$h_RzK-uPfyb2LmGO*JO7Kb(-6> z-ievmm8R_J)y^r+_@L^cZNjxX6GNz>f^(=1td$egC0l&pyjQ7}nO-}Z5p@q^ zj;~Btxn{OveR=0g6c0ju9+9~UxNFv_HZSN$c|%~k%f?ijxt}hARC1v=fOKZRU>yzIG?xv`0-sVmN6Jf{N~N?OhER?lY`zAOk0D|zW=-BgDpMeK&&F5eC_8R-Fag>20S--LR5GvXoq;~^dO1RNG^+Kjb`8+2ZNMFye%vV5Z+g1 zrz3?~F8k~7We}0V)Gf^T!tBh(FO3rR%U=K8Dh^h!rwSr)5l*G+5{g@SU%mJz&7q{? zEflzfGtPOG8&sQ*n)amH5;Pz;Qf)}ohjfeydo;gV-;KF0jFxNN9;rL&tlw%N?xN-W zT4*#}Oj@^}tm@$VCu@A)njaI&@(&YSeRqxhq!SGcns$onxmtZK)m_uE+a*#u^|q}- z>R59;HeIG%w|?~2&N{z&&cntWzZ^|t^Cr(lg!s`b@8w9+!Gu7T-*!Keh}6`zFm{St z!$le}(Mw(#})CO*925BYK!RZvxc`3D^chhTedZ45S_ z!@)1%^M?F|sTp}VV#5i`%Cwi+=i+vG|Mu{CJRF{2ix~Gw#n8K!o#$0^K zY~B=ZC!%2*24==O9tbcv6rz2PU# znlXL|=ldxuH+4jDg``ii!|nq^pTuOGwY?e*nU&I$i+kVdy*g^GdyM@oWk)Z~G)(3( z3){+qE&3)M9z2tR!i|UBE6g~NHQ=3;!OM-A6ck&oBiGRNDFNJ-XE5B02LFQ=PlPue zwv_ten_qy!6dv32=aai7z#O8bA|B-ajwf(2;vtd?d_9`GTB7fs2Q{{YVYt^fuRIt! z-pwLwXcXfJf~7DspL=kX{c)-3yKNPIVZ5AxT-F{}Tk%|cfZ&5w#1TVo-AgK20^(psZ{ZRAi-)%hBA1H&9QMaMVlaNI#s7)K;nRCu9KmIY+s? zk5gI)OVbUv$Xq~*DehVVG%ceDW^cc8u8O38JMAm(??QawPM$kX$znmBIGQNnENKd= zO0ziPcSdL7*Du~`_(Fx!V+KOEG#&@VC(^V#p2(6GThY^uWn|jrDbLD0(ah2HHb1&E z8k%OpCw)FuUI+eAWvtiQ)rJs!+G_IB1OnPMpFvfVm(UV%;I8dif?NLj@;N1!w;$s9b5<{N6 z`?}D>x*CbaX@7ubzQv!LtZ^{@g2>(u)*=#RDMlNze3`n&%r91SW|pklqvh)$mZ$XW zAx79i5x_dt{HF!8XFty4?d^iqMG<0*6#LB0(>-n6wb0fX$r2#SPGJf|bXnfq;@}6Y zp$-!(7G@-~K^A3?C`d^7WwRxVfK9cG>x>%a?^c_X@|SfMeY^hU3Wg=X>7c9&|0Oh1%U~zTY|uGt6EYPx>Y4=Iz`(QgssKYS62UB2;rsW< z{crNCP))cP0N!Ny4L#{v{-)9d<+a(Sb|Nm26sur6OR2k5QO6kV=bVP=*yk_hZ~Id$ z2=d~p3yG!sCS6PK*Is;tPze!oJpFF9ClY{!ESc{UYB#vGmz$nb6w?DSC1A8xC-3%- zNOH>+a@#mWN1?UVOsdAF;DJBP=J9zBn~wF=w=0S(77gESMAj3kgqbjh$Y@Rp4wx^3B9re zfhOu>jHkmt30Y+_E9Wg;DAtZ)Z5pk;6t;+-_uEe&Vl>EsH5tnuq-w@TPSaQfq+k*R zOFy@!eHv{Xs=HN+<>+F()Z-^Tf4X0_t@7)Ax>jO$;e#!Fp5jw!bK(fKZN1tyUGdDU z1nZo7+f?8AJ?OzSx#s-14s9^xd$&<8{N)Vv)Fe}XHfgcnYv@_74~&~ySZxJY^`eXg zOz9#l!u0%&Io)KuDlS8nnHHT{(zP36bQjZC;ULy`UqvUC2X<~O1IDQRhk!ml?)mt{ ze&$%%d@Z9jFNa*BIX!lj4+JtRj6k~&vAI_BjdvEV9j4AA^M|BZf*j-+FRw%3Q&lWI2 z=lJ)@GNFuKSDMws%GXfXzs7IlC+p?ME&Mocjo@qLd$}F^#}Fm~D>P!3$Y5byHaWKBHfVk}kw2PsvnGHR5jFd`)qV}dr*sQ+X{;KxnkOhNqc9rq=l8hDG2evcw2!?I7T;U3s({` zwzB4#Z^{#~bB*B@L^G9JqSvP0Y)9)`V@Pa*BC(R&XUU8lx#j$1-iQjj13}RTbrK>J zs(FYnBwmC`!<@DLvDD`I7}$?Eb(W1jp$=otMF@Fak_!s|OVIy*Z#vaFr1xX*&M30F z;m~!W(Zy*Mt#adVy_X?17w1%xc3e+?9`$z;%w!09wfA+fvxNZ4@gMh#_%KC>cHE44 zNp(Lx|Ml`tn$E5MxDztF!n+-(r=8NbvzXDXc9j<2cnwgo9=}izF~6(6-tBB#m$oA& zsk_4ELz{-TZmYbr?A9!7FaG+IN?i$5V*9FN>cayemTxdwu=r$Y2Q(AvUFLfCA(huB zCE=|i(+Ttkd(>R9f`UHy8X1w12d1jxM>QsGM~6g|9d35PN|b7dlP>nvh>QCZj)-2rh2e!_q?VH zH?A1Oz*}jo=V9z<(@c(Wx%A-T!GTr@UXs%aKf7O0lazk5EGFri&E@FdJ9m!CkFTBZ z)NYn_nk;%tNS&K~1Ldm(Z-uEH_b*gO9ox8PNJl%zKINpKIrdFo+!TxN0`;OfasS~O zipW;DYqS=sbF8G?J#Z4KEYpG&~<%P>refiV#!rXS@i06cx=WZJe~-eOZrL z2}t~GV}ex5B)!z6O`N#+hpo_h5k%94Z|8DZY4eWL7nPi1&=e!t1UD{RI1a5gY+qFs zB9Gj%USc#MOaij7Q9Ij|XIhUBjBmyu4|;xa9clZ}h)$|IH;O5QT!U7;;jUj-IiTQA z&q%2~%~ZqWWJ?x|c^I_O9Xmthw2Zi+H3(k#)X)|Ob&y~|O!?vXx78?FTF8>L9_D2j z7O8aefe_&`@6xEC_7HsQp}!jtY6qIjsQ!j9>dU(+Ue?)HR-!irZo7X_T=9AFE|s1=9^du3cd}aX7}}=>RS1{tY;j$x8)e-J)I4*eYrzVB{rfA9SPW5r8s8k+?pE+Q|=f-WpoXeRC!NfH@M7#0s z2Nbyus}FFl_=1i0NiiRnl$rhg!N#a!dUS`l39yhr2! z7}6J`3*SIYOi*A_Xr|jGR!R`9T^fR z;pYI`Ko%hQVWJg(UdJ4srhU{uK}E^xz*jGKsdZ49B>L~4_QQIegWz}0rPRKYc|9?v zD0` z{FI@~6qX#9`2P*-o1SaRLbF0ilkK=jPmEBri>DnbENmGfyy<%N7V+v>sqoZOq^u6? zJsAM^hNJMmN%CTsPmq68dKm-z}!Fu}g}*DOrHhD8bx^44fa6dupGc4jc3@I6}K=d$i@{cM@5KNU~t zl|Bt%?dR1D8=VAEJ0Bs#TeYx;$Ri!>_ zAj^@4<;pF$&ReM6nx`*PXQt-~N&@W<3f%^oix75Vi+_O-qMNQ+<1|l!3^C+@IpeVo z>qTTj7^Gs6@>x;W<_mSG*7-o25kFb_IlR8BYOI^CYr?(gJV8P3#{3omp_rELY43b91v2bbHt4dCY{o)DY;P z5RU18NcaCcS&aS1s2S5}<0_oqY{&zX%N4uZmGhC(jIN}3TH6*ngxMMUE>-~R8|@j- zx){qfc$|G`cB-qT1;#>q66Tc4rg+DV0khdc_xE~hq{hZE&-!u)EC`fumnrjsV$QUu ziGe2~5b(AjamxHd7oPc)TCskBMA#)f-VlVm*dReO*YjueAPWCgl3_4cI@6-UtK{!- zvAyo^_-EYvz{=VzOH#jA;PBchc9)1(I<-FZ0%=UZzuTovO`DFHR#Fpx)R7-!_&y_a z7thYK(<*i&3}CKS>geNU)ui~~=kIg1XtS*(YnIEdj<3obHX9SZ8H)L#J0%Iz)u#1E zA62{lw7nS%vZj{)ly5HuyTMseP6$ub1L3KoS~^!Ox+!bU^5Z+AwSinQWa|XRkjO&L zvvtDXKt>TD(M1{+0j@&5LtX%IOkD=v7A}QA-md_YO*rSlqn}OY0a!UdqU7Bb%O?It zJPTI|B1~A8qG>PIv*O5qq(1hJdssUfBKKT)?$g7#$j(FzW$j}e8yq_8oqD7^nr%PZ zDs;N8ge)vM9S^M}gfwXP&!?SCZ>8J&7gNGgRMP(JCBB<6zmiS`y$ClY<5j_#ech zy@6lCJqtW=s-hC`fnQA-u_SCNoPU{60Z>Os3R0{wd|)bL1QMn9L5wb`x=WvLNaT|A zc!oXLAsX(BVlUTf%g(vy6MjfI_`=L^^edDctINbdkKwNc1CKI_7zO-RM2q#OwFAJ@ zil}>l{-m;LRdi(6KC0Pge^)HmV=kwjcHS$H>=kU|15GZJB_HWiGuI;{KP-Nn>VQ3O ze$7nNuttU%qn7`nz9eevF=4noN+OCZ+*!D<+-166?~b$TaLC=I(7nQ0JdA3n=+BF| zOt;fn8CTN`P;3Rwrwu_v`xqOpmWFL(BsP2~3TP#v3ze1*-x7`s44Gzj+&YAv1kLzD zLQ&tOZuGDv)@2PKqw5kDWnZRdpJhD1!MTaxs1noEV{>TPX+)gSi78y825a(*)9wQ*VmNWu^`U2kq

vINhwD#6vYqCyee6x5YZagBu@8*59K(Lu= zem1m8)F*sEGr?NT5*EtRyH2+Jjxjmvxv~3@#lc?vHDvwV_QF6mR?KCPF>xgRyaXOE zJ@P2LK}I)IVekyLf#P?!dJtUm8~qmrY63+>1e3{vae@I7vGvXNPO}veUw}Qt9q)*H zyXel8#juJoqx*g=f@qySZG9?09N*lap16!0nwRKItXdJ7=azsoKA-xGInz*QJ};Rr zf8yZnpo%ZqnNAz`a2UcQnB(!c&Sz49<~e_5B{d~BkQuY%zJDWIQR-P%{2d&$jjlnw z@>D9Ee^A#PkmG1TQ!rlRgS41{1&e(o&reECTEZ;D zCfJAmh^O&1RX0j|MQOe`EhXlX#v`o_#V$Hr*EpMwI=f`j<+DN4EV#w1qDGMomq=(_i%iNQ!6k5cn23w$^ z{Dx??5|w26-;}lY z^$R#zGWYH5o0VE7`!I!_XdcnO;XbYpfSnpv_0MsjS=6Z`cP#rzYdv`_s_#6 z6xKDnO7|&Au{lyI_DZY-L0ZG_w?Uws_?VQQ_Mc5L?#j%Myd*96s|a<3dLP3lg3~k< z`3TOh+VzuVc(J~wfkmphf1b)&83@KD59HnoYLl6q^D`o>ba~m=@lvLThDB?vrND;U%XQF+`EoYLqmyH;-(U zgKk>ViBODgioO0yLNH!Mi>E`dNhu(ZJK{2o-}$UZLVqyd23GP%oY`?j)VINhvVHF7 zOJs|tr{IF0b+N=K?O^NLIW{}meH!jjmpeVdc${Iern{Xcmtq@U|GXcJ_gY7G7~q1$ zid~G+kn4;I6T9X`uATHMv&>Q!?tNMYfCuktCoIWJfTNBLrNYNmq43`~5X4J)q0^Vg zj?xxYSBa&0P*tJ5n4L2Q4By0Gex{#%ToPdM#)C5u&#PRD^1v*B-#bZ-V7Jap@_qTw3QDlj-h6UV39l}HSL z;@&J%H*z$0%wa@aSHh2+snk)~*xV%_xm?mo((x4PV6}~4$|I@byHXUgGMJHN()pI^`&RIvBQA!lr4Tbh-E#Y;VDSFWkXjB zfys?U$e9CE2E@7>_9I@-U761O4Hu5s@|7B>IsE3?pb-Iv_7E_bo+X^u(9H;subCY{X|N!?_tk-e850E`boy79%6d9DqP&&f2S4*vW$}5C??%BX zIjamvw_*b-RLHm3?tIITPHhY3%^7=#yV%SjvEOk3WiP>uIVZlNyUD7y!#M?}bA~Pg zYW5rmzP6!BDT?g1NcL>1g_vojW1W`sj>3ZwQ*ssqyVs1oOZ-%v#g@T97UqnBln@8# zkGjBm^IePo3=!aD4pWBSbu-0ujPl5?P2+IUr)S-jJFPAn%}Xh|w%6)-EgNf)zF)2D zj&*IernS36W98=*wa9lc&tpygJ`QJO`MvS$%*oR<(XWv-1$t!25|>p*^w;YQ+XcKM zMoCh0$R5jFh~0wPd-TTiCK3&D{>k;bctekt0ST~llQY|llX_rD-JXI3!b9)8iX&Jf z4$p0cKp5Jtvz~~kWY2eczt8L1wQjywj|h7(j_3xgn=_7_!*f`&4EytAwd)dFn-ug` zP_A#b+p7ys`scqt1E#LfKt)y4QDNR$%;s)v`0TapO+rf>!x%ZBbr^V1BBX#TDyCn-Gt9iL1hxuw#Ol7{cWu>XVrq@)=?Us~?jYBE-zrg>87psK&6)<;&eW!&4 z04Tr$06_RxJ*8mp?4sagZ*1!9Z2!N_Q#WYn3Qzymf6>KiHMHZm*wFmi{6Q~yBfS1AzL{asJQN;7i%T_~34KEELMg+>rZSYs zl&U8APK@Xg$L}=wvh3;!u?$6u%==571n_eNsB3w(QIZ50+_Ky;t_D9#C{->&N8GJN zEn2IyjIQuo@KlysQl*v4Vrd;YJSlw7cs7jJ0#xYH9wX*z0@m7z)vezQx9%XTso0ufmaohv~w`Z zfus(-#pG*0>q%c_;!_$d}Gtdw+(3g*9B-h$n?T!sK zQHxXxzhd5`0{{4@mj-_v!W!x%cTCj5NnNRSsbmS1xTLKjRG36yw7QFpMA=tLMKs1( zy234Sg+Y`o(h?wU=N_bOzf6hT7F)3DS{k1J3w9RULym`-Frd!HNxYL#3fT4_Z5}|6 zt&)vD3u&OCz#R9eU#eQKRST(y{@4bv_;`pweF$ig?8_?Ne%I8J%#2v&hWihX+^MnvPN%e+lzNB)APB7vy+pXGnOfuBsphd zVpU3n+f;X1HL>~oevngl-d6%Mjcaq30Kv7tTlZN~)8xN@<|b zgvBi~QGBgzdwP|A@0X=pe-nyqY@GnBno4bydF8Y)4fNi1l9s>9SRtcH%C=HM@aRwf z7w8F1nSOMd?`+AkC~#!kFmn5}v)$f$3Xh4)CA8M(+{-oppkZ$?o8SZQtZn6|%+?@PQ3u5am6iSpWo#R5o zgG0rLo!Fi1FU>3T&Qm3-Jp%k}^kX!av>5)N$ym)uVzP{HBp<8Z4!9CCc|ckFeALbD=xp zkRUT7>eTnK@hj_$eQ!~3H?-+pp&p;izjuMSOK?OharAgG80e_P8LZG1u|L{kdQd5Z zQKa$dX{UcUj3~98wx|?xBVKXzkR;glh_H2Pt22BH9`q>aqfn#b&@bXrwHg~D1esEz zi$pX2LaBpK=b3S_bz{4<@fCw(=q3zk|3+&rfXdXil{vj|`E}Ch1kjLfE4c6@xn=z^Zh}62}CiRMe7-g+1U2+{tdko9{!33=5SA4alk}` zIHKi|&Wx^S+v1QbU&-)2v;AuszR%f9M3cMWHFZe#ardy3-OcVl=GuEa8_79)g%jJi zo%U7$Akl}_cH-uz<#D$o&SRire)wRr5QoS6HcRPcEP$e)s_aAW3y8@9HKj!6tNY13 z-FL@>{L8XxG}r*$XzhWO&JulC6#7KSfA6tnR=EG5hFd(pol6CYV0+$X4!jb%L`O#P z7k%f-#*KQzaNNO1!AfYZRtEQme36U~vGW+tL4oLU97h3t#r^5l;3BuJ&<7T##F~qa zQ0>+O?X5|2hw|FPX(rMbyBNEk`{{)$zZnb_`1NjjoQFqbcUIxYV4#0>4>E3oILXno z$h*#+fk(WQvpqo92`^Nxn)>KQ(;KUS|5peFoG}EQ4xfSQc5WD4R72R-(Wm}_67vjV z#gPwp^8oUZ{nGLj5}#%JAD?#o&mMbv!e!wLh)4eCGknMM@9oMQ zS}HA4#lm8R zj(P_1dnKSP*bpq;Gs>=8(y1Ag6N8l(5Rt{%BB#pgG6j67xUzrAtBlpga$e;sqR>^&f1k47yHp zV&8^UD`xy{6OB6qnrCAR=}rjZB?qGGGT&2m7Bc$f!E1_%xDIbWsWukcH$ZH~l{9mS z4u;Xt+eH#2u-Cab!1~=@@6Vey>sQ+TaR+3T%69qIc+y8lK))yh(WrSGmBr@e{JZz} z9>NN?zVdWd;w-8WiL?CNx9~D|`a9}hzv97fA|42@v1exV85PDk-Qfs;{Invr9OQy_ z!mKxwh-;&M1jcLnOp88_{HwPRZ~HlddHI}0zZ6nRv@289bKPEcx}r>HpVGg1y0bEs zs)lgpZ4J1<-*vj&`4tcS-2;pGt_7k&Nz;ko`lifdTs+X{lk8Z+DQOGESt`{= zvW+dRG}VNhQgjSRG6Bb$gERSmx4RZkq6yAi2{n8iEPmt)jzG70nJ~s%rTGQ zK6(0V!9MSe1i$IsL2DQJ`K&>rDA9wq*|S74{Xy~j)jvWy(@c^u@=47U8Fgl4gGO1o zNQX4p|H~YW?9dgA)}hxDBh->=cOo*Yl7U`D?CH|~OTEm<5!P2sA0^RG2|nOvD1=(i zq>#gl!UV;t;+^HIXgS1>zf=r40QW?ajflqFA>1NEvOO7DBax|L544}T};iC?dG?BDFB#M9oMJ~@7Xgr@9-0E4W#*XChn zmxnUisi3Ra!k)~L1#ZMS>#sjoXBS`h+uhaat}yT#dzn}5M6$lz4?vdyFC~U0bxx_I zT0k7>Yhl!2Afb#(;v^4yz9dw>X5Cdg?F0>NzSzZtAIXrn>UzBsKo*mVYViB^kCs_M zda(NGqoNt(x%aMBY_kcN7QR^Lys17TESU~2dAwuFc;X~hsQEnyrzaH?CyhZM3lx2F zPlpLfq;eyxlCfeLv1WxeYV3ESZ4dG2ye5r#)ef-gBhP)}x(K5x&?w8qag4?=F+_d^ zBFb6l`~_)+eu+7itkXXqq>cc0Z7%wuuu?@{1sAd;40XlyibM<`9t1Adxc0v-2bx2W z5xjB2pjX%`xJADk{jZQaH-RdKM#BSda&>idb>0dRJi})|mDHI*9Ke(LXpchFl_vEMRCLf#IlzA9wKr=Muk}L zf@R0!_TZk0p_O~5fm|}-gPP?WgQyJDBWQ+j%p0^xbh62}k@Y)l`%~7yH@UeosHSMl zKL!}2oez#200vafnkbp@i&qi48U6HBySA^x(JntwX|T(o{)PYq?<#0Z0&q@Wf7K5` zatzDDSk_SS=Ns0;H^-Y8%G>08mGZf(+c2*{T+GYIPnY3hT)O2U$?v=bQBS?V`V++{U6Zeb-uAw0X42+6?-|pK3624aBZIa9tN*knf z0!7~(X8BpVhfFq|2vHbw2AG0-S-KPz7U(&D0c&zJ-*ZA5s~^X~xo)7C050#VfoUL) zL7?Dd*?N=D2qi4W^G5-mPu|!`!;fJn5d57%Drk z$nH>Ld4z+_4(|%i4M7Fct>q_KMWmj6av)N_ros(x+1czFc@FpE6bcLT>c9OF~4$+`xEy`vi_xE zpXTM1aC9mBA`!$BE3m8xo|CfyqcsU{-;_ji#}Q+SvNF1Vo+Soh+uFiW(40y0F}*p) zOP%-jCHf%n`scL^QmfxC9a_M?i$BV45pjoZ4gHLqT|7UoZXVJ?am7Ufw_A^0>Tn{fs;#365; z+&lFZ-$W&-f4Ml-4kP}npeJXVUiV=PSV@Hw};}C;}9nf zV^>%Fcmx7%p>WF6%~QxlV?mZHlXm}B0JAN>h@X$zm?Ix`lZCq@K#|(w^JTwIiy9<> z$6+ci803xlec2yI929Hy_VwMns;EVNm$X|=!2A~pjcYv===+~b#srxe0BWorXKkbS zfhjaE806Izgbf_c3n?&M8Pa#7rA0>&8a;tb==KQHw(VmZA!meG!^`Rkjv!}|DS*tk z%FP1%JR)=ehfZO0*{#8bmT8w1G~nz95$ZQAdKp87P-M;(mRo~^ZJmcA#4kdl=ug6a zuSmN7w$G$S74Z;=*6;ieI#MA3NNq*IKhRG#uYF9nt+;gKX{AJIE9w z#I>@2MXiD08IsFptF6zpI4c{t6!7b9d9}5L4Hq3V0a_>>Uu`LncmUmAoki@YEg{)( zSJRn5g09NH|BT225bdfKPTPvjR3DA>5)>WKg|{cwV=2vvYPdAY1=B zoQ=@>pvVH>?0)qE{GYDd2l}TssY4x0Kh+JWRP`>m6RPU!QNE8ik;Hj1~_NB3@nd~j&un8@!vn;`q9K8J}YyQv#ov31=Dzn zMqnM0p~g0`Y`Fy)s0~&{=nk29MEt<8_U(-#P4qe(x-Rej@G<4MMK?rLMG|Kba&rB?9HZASY1sZNnuJ2Z z8k@fN)a(9i-wvL{^e2F_i>e7dSBdN zbn?FF_KCT&2v|_%1hbz664B7fg!8ORG$HT%Yq*S<>U9(_5<@Pd9#5qO`grs_u@YYV zhzu&GOH%+z997eSpm0_Jn~_Jc89+L-jNw+wX!CW+uyPuD z`P3-af6+wtIq5H?7FgpNQxs!aD8(UAMQEpK;6Oo^RJkX8E*8s1FxL9%O?hC!xyR%E z4f>CE5x(TWBswS6Zeg|rSydbEJh>@?{in@IwH31ps6yj878$3<*4Andd&F^2ro zY?a7VY|NME<8DFv#BYjww(}?_UOPJ;AGNrZ5hbIA zNig$%yZEYlM|XEXU>)p z6W(N7#9@gvA?*4uyHHeeRBPXP?mP?^5^yFja1b1bedi`i`;ZQQls1B)jnH(@RUT+n zl(gVqGljHzw5lLj2AN4553m!Hpft`H&!99b=P6j|)dLh5b8g51JSZ_8*F?t(C>RYJ;6F`V2rSp}f}BPa4#yH*IPjYg;GVjxS_~7rPH}HBe||Fk6Uqf_io(X9g4fVI~0df+}+)EBgI`dY>GRjxNGrU zinUx$-sGP1z30olYh@;Dtv`Nwl9`pvj5(e;+{Bq|><_wS`52C~vOTRMmYUwf7^u3p zA@0#cUsV*P9jqPDhWZtgWy+H=0yO*UBP@ioEOV%k`}KlPbyKT}Q#cB@v9%_sOw@)) zP`P-@-r){SktxD3=4Y}6Z|1yEOIP7IN^;16N zu6w{QoMK6bJNyb3B0KUTzs1q0C@7 zzN%ALUOiW7Y^aQ_Pj^Tnryj%!-UT4;69(c>$m$WM?%7cTPeNgoNb}5=6vOk&_2SgH z>|0<$hs{{SW<0053~yH^^=oVPTBgh@fDxYzTQ>Hpk=8lobR9iP9+a7Xs$UCCd?uTX zb?Z9TDkAPDs_`YjEQa=F+a*OGt8^nB&R53Ep*5Oa3r~5=d6Y};90In*s4mam$kl*R zl9ohFg#b@Z7aFb1PAt0M7*X+Gp8YI}R*jtmRHf7wBP#n+_fU_z#g5t?*=Sx*4p8kY z>Th-iVzb<3x0Ny#JY~I3v*vh)**Gk9i#FDX)(aSTAN)Hz@V1T6=_!}L#2w|6^N`^^0&}ClMvwgJMYJ(f)Wbg2ID&cN!1RLa$yf(= zFjVWN++)*on0PQpb{;+-Oq+TO%BiUsd7oI%V;g$gUW*=vNziLrTt-BPEp1fW2qoa8 zat7|+F31~Bt8W|SP{ft=mtuvnXSQh>;nxnx)YYJL;YN(*-0fJCd-nD@l<1+*H@4Xj zH;1=VzipvGvTanWR*1GpL5jB5KeY`5c%u3;^gx;{KX{by#hVkyklTUgbz{rl*mJu; zBnegXaAhm|ZVN^9Ty3-&r&;?QzxfC#KFn_2ZyVklBiYJ=v{$P z6WTbT@g+M#7A?63p;qX`RMzG^06?N)c)@j>f-FG2!T9@JUw9$(7T9YDb*^SiiQNeR!psE5t9F3U|RnWU$OjcACA zx=ezF=K_m$pt+o6rJ2A2V8nP2?XfZ>MPG@*^hn|mQfjf=#>DCXF~O*d5)K8PbP(XQ zNm&2h3P{UCo1$CU9!3R^=&&?ghl*Y`=;=gC2oVj=%tEBKmA+C+HuZ=Z)@ucQ(lP z!R;HbC7SU?>;t|rgaG^EYJu_2>|~Sp;x23n z@2Kxn9mGPxdg?)j1@-CG#wHMaHF3I@1-<-Cjwx{rqmGgJtC&@+@&wzV1> zboCK>B~BOX;xxQzsFH#GRO+d5j!tj`@1?nBh?rcczFS{7sH4;D0BSr7v@KHeJcBRz zktv$yAT7r>U4%xHPrJ^g$n&ei2%l^w60xG!V-gQoA?Al>7CAN$dU99f4{u8;OyyP0 zs84wdX7pq)K*W`sY*ez7Rbbo(jJr(f`$o87p3IM?=NF@BH^-8yp(Y&%xPh<)*wI%2X0_H)Ho zB7q*D6EE7F^nnh)&l({&LZJlTHqNKHy0nPTXVB^`-~sa3pkNgmkdeRxRJ1_p$m3Wvuadnz|Jaa2P&P~ca;vanBFxK>j<8Na{8Sb zH!pbf;-5o*a3Z2aSaAxl!byaF2*7y2c-YMLEbsE~Dk$&~wm?W;h9vl@1C{thlWvE4 z46AxcQ5S(dO>T%I%U8njt{2Igh8@VFeMij-$$d|v)fAT}+_NT|NE#)ZK!0x%`G|bC zrNy;DzGqo@{;|>D^;dK*Gf_%j`%CEGO*Xo?6Qyz(6r>^)ctkp$Yok?y8qvW_?WYWV zGQN0uBz{*cVMfhsv~gR*grxL|`}f3B)ChU%6-v%1@OUenIa8=h)jArLd$0RT{9#_Q zdXu&Fcw7nD8!up!!xsna=Ir+lm|+7ei|+5jQicJBMQKW0*=yOak*NHCUIaN7HJ0m= zX0j1YrV(p;Qy1zQx_a1tyGe_+65fh$O9dEqI2qwC-iwvkJK6M1#Qo7ZeEeRyA@q1T zyrbLkBhhKHy~El6!pkYjPJQM5nX_zP>zADH+D}@Y_f8YNfRA<~S|XljJTKDB4V#0G z=viRrlW&TZ@PD|II8VF%trChoLxBNJep>B8QoEa__fB9ZBGzVbWTQT@viP2Lk#cPK zHaZD@ORd1FC8e%_HlCdlS2Y>;%P5JdH?mW^s1j9V#JbM(O1bXT{lelQ_ed;oBI)BJ zAVVdPh3mT$3~S12{x|;Is?rmp#=@~x{SMmA$Y;(`-hE8JwaoG#d2ea=rOfmL$}Oj= zfTtdknK>^M%Xo*7w%tsRxl1^i0Cu?LvSe`+Zb?Dt;N4K@ywR_c-UCgTSfQ-f^e9yo zu_X`pd>xtV{{hTtPtY z0@i(H>bncuJo9t0-tz5_f(+|tevHO0+96EDGt%76 z>4u5_5w6WVAE&ZPn94@Ea+3_gCOl_%kl>{nZh#kfuXj0u8X2tjmE5YTA@#g#(u>Xp zHo+JUPSw_yiVe@@+w(O(j&acxhGE=1;ORk8hpO4ule^BPuwxW*zA@e?DIEhRwO91x z?)8ubK8E7hibJNd`2GdCcS72cl~m3oW?onkTia{}H09t$1wmZ9aW2QY?-=!_Mwe-g z#@|5{af?~mMUcnQ5Cwq9Kd`oz{WKPt)4VuteWJSaAX`l$E~ggiL?yd}`P=59=9VZv zwlu+D3%A`O|m-%6chmi6cp0ij)=FHz0?1Vw^A_@~2Uas;XJTs7l0XGPZ2F)7 zY{80ahzgFc=s0UH&_nk|`Is&p>8>ZtuX}2{Ox|iN@qN_K0I6_%&yB91(C%6#NDwI! zuFTj~1D+`wHb2y#UH7)uNoMMN&oe5_YkE7nluxWrwJk-k9#=aPpR-r{LSeHZC-38p zXUWlS*F5~z{cxR8MXfw>UF6*#DgZ{YB$`-T^b@&{oLGYV0*efYa15g-au5?iS0FqUprLZe1G(oR*|-P`T91U5cQwq%S)V_H0C@< z_umiC-+yh3Vm*0K7>-}czc9AMIRv=c`XIhu#C}tpygT#H^C{!N6q}4L9*?7oiA|vh z3FPy8dFY{_@3G8$PE$n`8V`x@*w`k=Bx}E2b4h~%+Zz)wD*IX zz=rGS>-(a}jfzz+l5fYvy{m4K2mYXVVqzgs=}z{N&rAcyTaQ=wgK9!BqM)#NeA5jk zjr`+pkR-QOS0B{ZTr|Tst@ud;liTMjus$-48WFF%pro-JD#qR^-q8a_CYJT4G+}Yo zo(A=`za-xm2FMs34QquL&woNpeV=2J9ezrL9=$e zrc0q(8m?=tTxGYJV@#n5xf?$p-$#0{tE4UOUP?0ltc)IK&p!vqjABw&tfn9k>hcNZ zZxLicvucmkt(neTe$j3GoYrJcNqn!qD;^G%W)5b>n`hz7w#?P9Q4nBIvZfs1qpSy-CA7;OFGt1~_q0DciYb=$ccj9_++C@WXi^+?LDeLp8leXrp>c&;&K zDjaJ+=>!nM9SbgO!onG^qALILWzSjr%kWTg zb1uK>zQ6eKriq19Es%-{tl~x1Y>xgF7OL%8q_0&z*Y4O6C#F$^F~`LGfOv3*3=0=8 zejJ*AmKZ*mP4BY6NkPzZw5#j&gS7n^AwgS1*Qs7VPGxhpMW zs!&EP6uz0U#X%dyV1c&i>!15UR4ni18^kNMv#MYmHRyD;mp(>44eD!!me}910Vb=Jy(>A`OVagdd_4Wi%mz#P?``oQdI8F>}Wif&W z8<459K)4Ehr%gj-={#aM1Ogo6)zhuIxIf_)o=l$TPAoV2>O@3{R!VT^k~5$8{aSsA zqPHW>qTQbFr)8Pr+&^99v-6{cAnpp{&wzFvvxDXi0Bvu&f# zyFEKS6xhI|5&j6p3by@}hcod&&u7w60_>^2p_-1_5Y=zm2Qyxx$! zRCa~EqF(Y2zpcS?Z<##vR7=}Lhb>ffHEkCv*ce(fetn)>Z0J4RYo&n?F4b01XJ?ee zPhAqs+n&-Quj`Eivdex;Z_2U3%V^`xeh%x(?afs`8fVsTt|`& zjw+e1meQAQ-Wqe=ZWhgMC(Y+@o<2;0`~Ywx=~FFw$olS+=dszfT_7me>Y6EeZ8s?up#FiHT09g#>g1MwlU{C*j2TYA^iJPGa?g zZj5OuW}>`WYspritQxQAt?jIWt#3xh4rVPlR*1pCW|VD?IHOc(_P1>bzLMca=N682 zOR?6VigqoE6w3-%TtmC8$Zz%rY0_UM;Bdr+6x6U2ZPxPhW*G4RHWcFbjvNEK1D5iG zwJD9m1YEeq=XRaBn z#WiR>P8Y7Dw-jV_O|>54{mLe^S4e7RW2*zx2TI^272Bls=81>}KmPy?bOJFhlzn_R zQb9cUo-pSYN5*%rYgZjBTxWT+O;Jm6B~f!iL4hxcL4km|usK+@Vhh?C-cffZPKQ%w zQ>n5iH*ob~ZFlUm0e|(R-&Iw zdn82}1xN5d5EM;yR8lm{iYn0JSw;o`EGQ01uZ{5g`HQs~($f9uYhXycCsk&b_Ra3k zcg??iU1xLfwb>o;EVAD@yCl<+`gYeA03$luf0y9d0ug_ASq@9!(|ykWKFfl(vG0S~ zLHt|vRr)lwu3$)_(EW}Vyz4sVuXFGezj8qAUNt2T+V-SVkDuha9e@=%9qq)!I z+LF3Nwi-(r&JwX=#Yp)R&?-h+Wt=!d&WvAwu$&Q!K{Ar%##MEsHoS=rC3s(OQ%T3n8Mhy!x z{}t|Asb=v^tQ~clPL4+Rwzys_reU!SOMY=$QQbJ%9?=wLYi-_CuR;59S}Qp|dAaKk zYbrsBpJ+~NS5W#<%-Ms2VQ@6Wvf4uDlh~!HPlILcvzY!+Cn-xeL$#4;QO zZF}|53ELGA%S+*6v$uw1X}D(4&X!S5TtJS$DbcfuF405e=PTQUndr+o3U%r%M}eDf zJ;e)*57?W7KuWuw`e+?~n;g&Ha`ZuaDhN9iu*cEaKM!$iI1eI-G#8j`-=6+RhCgEU zMX`}VLXn%-go=x0m&=oh3BX}QeW`eI%-JX1P~P<7)2Zz}5bs|#;uxN_^oK~O)8KQY zmJxQa$LdE$iG>pGQ}(AA7KVsR!C%%WLa4!wnWmkWH|4dWtRUMlA;5zaOcfl(vuAAZ zV6up#Q?%9m*IO!DwGoi9jf{O-XQx4BFqq~ioj1I-en}+aAI}OjW)oNDCWLSHgLz7) z{V2xj#wB$E(jK7gPpvKv^6XzNHH91J*xq{xDsAe`T1o=x8s`<(U3Rx>d=R5FR$;wb z=6LEim4uuxT#%B0sUN(S#fy9(d*9Tr@Y{YwsNq_DV2Ty1@UC^o8d8ZE?r8MBpk3L8 z<1tLpTrMu|KPo?peu2GiCqS{^`x{ z=zhsaGqGJ#nDRt-ta(Pg0M=pmk>L>@4lE?8x@aGIX9^yw4lmfKpC6>*g8`HVpH3$+PVO+fg=9(j1 z@jbI>&2XeBt?=Dv5FG}ekcK?bRnYFVvGAg#OPV$hT9Q2evlOxBs((lTiJz^XdgXT+ z4aJ^|;0%r%dJgy7Wt%)%MC*pbj0ZL2d9_bCj$xoR<8*_5Bm-l+(=Tb65#{|UM~L}# zm_V}x<}gq(3v!W_IwJ8?ZdRAkZvOl^d*$xQ+Anw0k-0|RR3iku+pgy9>)Slr*Zkl6 zk2m9EmtmbkY#_pOB|X_+IB<7bhko!wQw`i1N8K01eL0Dz$269ik|$QhflqMr{(4bx z@c4>sg69KYB5e+5#@USbV+C`0esi#6BsVFLvL`mMjuc%FL1%K@9UYWWZDoImg(Ec% z7%)QKPpLRJ6Q%p@O$m64tD3B_{Dp&cu0As9PC+#jt}0mIlfGPj+%zBLgZ1h4QJ&HI zO2j9sQb!EPXnkB=;pcgpuO<0oLz{^5v2d8G~b6qU@V*cy1Ax8ZjmDg*Fy> z-|J)24V!3^OtF}~bG@-SiSzd7X3hmGYkcl1+&2*c2@2ioQhcf!U^!0)1+`@Q7EllM=4X7npx;Jo=wDB= z_D+(-*fCJS3()sd6{mX~>UtKT%P5qx%&VN#GJ)!=mvGe~A{I^!~ zH}H3;Mf_jDe<(L^z`vB6|3`)YmR9}-wuZtbK>b5#d87R$wET+(kMP%QBD~#@x5@3u I|Lg7l0J#%HjsO4v literal 159848 zcmeFa>vkJQk}mq2dWsrqC;$=(UL@bxqByif*__dZOQP)QA!%$7D3WM_0E_}e(w6s+ z{Wj+X&I{~y-s^l{T(YtX07&bX@y!18S+`83hG{-10%o>d(*+{Cl<|?l`t>&=ZPrhpo zTb*X|UBCBBvzz`>S;V&|&0g;^dED=IJ15Pay!tln{jGV{=_UW(Kka3GefdK(yX+oR zk{_DyhL^3=@mneSV`tRwwv%t0!^^I{+HIbv|KI;fcAJA?f7Fyx-=)Kow0oKCoDI`c zdGo4&@_+xIVW*#bJ46feYWK9$?FKQAZjz)uQeR+A_J`}q?POP$SF%=r5L(hoZ=_P8wbpAH>CVffnI-;vTDzgC zPzdW+Fy8n%OU_aItp6@;C&$D7Su#3Jlh#QGuiE2onvDAL2KbN){(6{#s7Kw)x(!gO zhC+q|?_xSkQiVO0OBZq0y*TUEK`m)Iy%~}$5chqv*Y08rM`^czUP60lp}yV4;_i+1 zX0#S2;Ne>Ri}mD1f0Xo(k2~O6UGR@=Jm6fV?ZxD1JW9IhDCvxn3~b}poD)c$KdP0f zJ$eI42MF(E83?Kl*4N8A?R1!o2PeZO*(p(OWbkMMl<-9YI>Q=Ius)Mv+Qq7ECn*Ug zt5?8bvQfKnnl{l!21fBKnCcs>$fd0l479SIRG#;L>2$lz-5GK z$;H=SHokoDmt^>E9iQr}wdA|B^|rqZCbWY8ttL-9!}Pd+!HV@&;e1;y((KaXQ?6EN zeCPGcrLVvK;+v(_MG*pj`FUfd{?+1L{To#xB%}0Vv`kjA*c^aOw3;JuzU2$Pxd)6D z-=1})jc*q3E^BLFN3AVw_f+<<*%L6T#RQCrsEG?bYxYjY;O8uq_Lj!kV%n2Nzd)(q zN<{KEh?;2>7HA>?QGB_Wj592XVdouaf>n~LbLNOE4^BI$&6DM*SS@D)U}AHvE~%FF ztNo+DrL7Ua&;46A0!mtEDFkR6&lpnD?Kj(5HIwgioZzh9PBHehT6wegbm?nQz?_yG zHal4=jHG&&oh&9DturU@Avn9^{A+SAsU*!&GU#U=P6D3pCD~|L?bPH`|5)A()4>pZ z^A%8tKl)<7vwm0${bp}FgV%#*E3I}Glk{SM(LC=D(~Z61I5iN?Azag>lRU(t>9Ij9 z)+F2S93~K?mE>S_Fg)mydGfzba&IG9mER!zyg~nbjE1lsx35L?``evJl0i4 zP%HDq%RFgL9uE%qjDP*bGM|^j;}IUmy-w?NSY5gJy5^;huxr<*9R#rf{Y((C-!0@$ z01WB~tAqHEDqNW&-BhAt*(iP@+J#P_v^J;I&*fl99x~~Dyb()6^^;141D1< zF|tETr>7g?)_|CdL9n}+;j`(5HAL_UW$@+DWwEA5d zEzTj)`@3e>d*_gKxN{izKHvi3B`9zWG%;Eo5jKZJBIew%NYJG)VO_i$RuJJ?lY?t@ zA+mtI7L?7)A{#(hty{xk^D@wj63M3MryC zM{FIs5ad)9h_#pHc?NC1qHKpOr20j14a^N(KLYlWXNyAb}XTvh=2LMUC_T%n z)`-kc9ct5Xl%2yWuD;Fno;i@MNPG}7h1uKMV)9N=9H~kbD*4UisNe6VO=xp~irv-1 z$O7@a7@r-b)Q}XhXIQ_W88ArE6snb26oq$Q=JEGiubyx10xzhdfyoq<HoM;)pBjtWxYo$&iGVPUGu3rZG7hwK~x8mM(xS5uBuuFrXyE?72lJ3$M(f&*i7JeSv%?An6Bz;E3WG{L}wC7HHUGpA%-&Ckz&2U8OaMn3`$HMZ)vZq2=t$hH>lDUy!gWm_0^BTEsW6qB8KfR;E9i0b1`*MrX|G&0lc3KA#zWLcBj?aX zvihw#z;jD%p-Z^0+KnldiB5Hm%+2vLChN-1IHlvt-uCOgN?kxmtE|T9R30nyV=bAT zg>NOXFW?B8ZaJ%&31Phl+qVj9UGSc}(;ArXUIv_I4IoB!q^)sKgFay66d1jlYJX+%eQg%Zlu%4y=hj@AUyp~C>gKq1yLz#C(7s1z ztSx{DB39gkbZ~Y|_BI-Vw#vK~<@H-EiN=+Sii!%Z0MuIjl4KVe8CO;ovI>-QeM8cN zv=hSf3bX-=D1f?lB5Zav)RhHuM8|9PO293>lGL@-<{f287lLyL?T#jVnqjQseF>CX z9aruwWPc$*jt8aHtl5K$;Fq)k+JhAU=Oz9ajlQ;TB(Zn!V!>C)uSwRYXROs9!#VGy z1urAmAsuQcd<=tv5Y}aFYnwok5uHuVoofBwX6-MSnSAE+^#V_ff)a+H&vgv2OK$|J zhJF&bXtQ(xK{l=|&|fAiFpR1X<*Vx8EGB$x!1d`P+#AYz<7?q+Cs}8dnd5bxH27B zdLE@;s1c)dkQ~87iN6>ph@gw=l`KiIp213+N9xIyt~;Y!bKucv{+Qs&5JVDj7LPkU z&`3};7eO{};qM|pAAxURA4lK^C!2FV!}0CVGf7fY!7v&R&|wv;C2c|kV0Cv$MDNlD zfp2i;YtB!MZv$mvU{JQll_gbR@g9<-jwv0HONyJ~ywkhCX8Nr;kI%Aqtsw@6|MVy3 ze8hRIvNl|#*dwV+2mLBY0W|Tl@$Jrwtyg~)8E)$&Dh$2HXp{wXB%n}Yu-Z_W@ze?g z@?pb)e)6>&@DR0V-v8>(OL1B)CNE#h4>SRROfGOc+36)MvJR{myz67d16QO5X#Vim zgINO&l($EnHYB)+f($jio5F(HUw-&#eg5G6!REo;LoBIrc@Nq>8wjh>#_$B>kgsD&p`Oy~MiTAMVBZNoZf=n^M!y6YZc#Sm zH6TS}0TDMIH+A-wU%VaZ3`Au#7b@w<8yP6N+c~~8ejC^=JSCJpkwXS-a{s4;^MiKd zVCnE4XS3Tp$u>~p&5NDKFQ07Zov^aV^Kqqtg4!sS#jrB~<8P2HQXg#ehuB6)+pnPu zi?{&aFuW(Pl;|+;NgKJ{o3m(KnT6V^OS$iz4_L2wGI591)w{Y5|;3eQ| zmC*NtN^(vKn`8MT_S+ae$@CuZS!VYJy{F*dGi#Sg1e-B}f8JnqfoCWv^`a`#@hqDifsX1LaqN_{uQ~ z3?DMcwh#-n^Kp07q0%6;qSTO_HX-G}$NOZznhGO## zmaxPoaJ7t@_F=Tp*1rJ(< z)~PLyPSLE%)Buz@6!1kXjH$B>r2w|Xgm%**I4}sJ*N3qVnm+Cfvk_E&B^6MEs`^;XJ@Qge zIHNQ`EJv89OO%5Y9gYXUpyT6q@@;=~O42&Igka9WK$5;8HsNtA;5(d8VNn!>YyDK(g-b+B@T_ z_9RH-S0@-VR?=*Ms0ru^5ZB`JkPa4sUyxa+2W=0^sX8;b=w3o6Plr!Lo>7ZNfEvQD zu+ffUV6*KuuqCtN7LG?#fc2!9r%R{-ISeA}V2&Z$Buf~2FhEy$6l7{}26Pn#P}QUJ z^oTNyGFOZ|Zm6LM!@iV8$l#jq`dEE%%=94W#ZvCH#@*)7hYqC;Os_xeoQQ>HD~R*W z_n-_!kDP)1!zI-3VOUuD7!~aBUaLFCOO}#h=-`rhn~rvO=|z9(aSqZv@rW&Il_y6K z9cpgSk_Qo|@Fl$!c7+DPR_IWW#mvco-XsRT7{3Aw$+p!WbTBfiit8xEcnJiuK}k4$ z+rusbTw|kU4J%wSl2}sw4zp#H-|Dr8>3Q<1f8K@6=QuRNkam%940eQBJwZ^9NJzxU zq_mr8Kuh$}t+|1Q-L)m`&X&xra=ZmKVEmtRSfvYAsNUU*rbICCO9C)-3Z>j)vLvsc zZ~bTE$8UG`UQ-QG?wH62tW`vJ;iW4l(^{eK`EMZIC;)^2fjm3C<3146I3IGW2mu*p zRvVeZmK$~#W$FcD=nt6u<_OUozQXcGpx{9DA;!!(W#gJGc~v(U3hH5z->{nLW!Opv zC2k0)+5`hK&=Q8uv63WMk| z1vAT5>fhlXN=xyJK&#z&E%w;X%ew!lN*~}J?P?7k5Bg~~DnsbpvP1+hkv2Kq2I4>Y z1x^-&-=U1&$WjR3NpGRiI-;TgV)%$h^mp5r{6`Glm_;;xA7DOGm72y?;$a840MqX^ z*gz|obQjA-y(S7rgQ?1Ih*dV#o15>j4$0`{LlFPD?LwPH

alp*Kc1()kEp?Zr)a zK+Ls9W5^G693QhQh|%O6A=xaV+?$HMSwyr#!2}>sKlm*qEPBigL7W`Ro{v_b0kPRw3>&X_xi~4t6RYmNzN*sqBjko)0)~k#XI^2^} z{GU2LHg~uSlTg#+_MY6B1> z@;&->IJAh0AkMk1c`~aLs_UQ|ffk-`QTjSy1lV>G%@ZbMGnqfbsR&U|N!>spC}W-r zXv@uAA2yXz@)mmH))W8ljlGe9Qsd;dn1(DTgYgjaP6K3)i{czJmRH=hH`i1|k8%hi zuZSMOIB+gvT1}s^TXR2;JLxE|1$dwIg<9b1>5bgQBQs(99^!v(rI3wzwWs}Ii*^I5 zkYIJ+2kRO`!pydCFa#TUUc6LTzJ-FPh?MBTgTTV&g8q$*&MUxm*Mi@f^MMebA{fy( zFHRN{W~NkJq4pA}0p~f4rcnSq;Kr#dTE?u#89Wxg;X~#0{{Ty@&q<)0gE~Ygg~L4X zt7zHpvHQjVh@U}a+?KcUSOBmYPy;+@1Xq64@E-O637yO@J~OO?Ln9RGb`(n?(<(Nr z*{DAd7ngA*clu!^p{WQSAp{lGNj9J@Bp>NjP@lkZmbTDSuaC{6A;l~_?i?La_stf!^0>6fYz)Mnn>KX1L;6ND!bVri>!_;^IPlml9su4tUtpJuI*=`4HB< zI6jS=65&0#E@9NdPWHYFWKNdh1rTF`@P|3z_Ogq$4kK!)bF^!3z-m&QYymgrCxq5S zr4a*R9ZZyF{Ed_a9kfmhE5c=j8r$++_&!$+r*oLnnHyJmC2ykPf?hN(qfoKiKN&%3 zfds&LAo=fE|D?RA@zGg0=OHX}$_0AD^RrLUU3MlJMy%5 zXD(n@3HPx3LxjdA+#SRIv8|x-z>kvoRFRc&hMwU(+7i_J&M`dz=6c{fz- zI^s@9GvWm6$&Vz~Xz4~DIjhYCCV5~g0+zKxQO#2p0@wAeuy|cgbr34uE+UJEYlWd= zx?$MhX2eFqZLP*&5Pmv`fn_E3bzU9;9u5B*HN$vF1RKcst?j7}IqhByH>l)~VKvd0 zJbm{-wE<$@lsFaa8fKv$ya;@Ikq^RNIIz)B7>XiRN~1iQ{7Z2_`}xK@B(rY9gKiJ^ zbv$ZWM%0dNNs0HY(WsTK0^aopr1TX3h;rSAI6bTKuRB5Hfetp0-*Hp9@?v?bqUN4# z{An$ggq|SI1FgDYp^y4qE7%vr=hs)HDBPS-(7`-4WZ(CkU=f7(60`#raazZ46-Jzu zSXp~eUy;)U*%)WcF3>>b=#vo4uTN}X!GdF}<5r_F4di8mkc}mG@6zWg zz|gidTd%TEUpv-7qiSde6C!=i@HxRT+G@Zy;t@ZAw}DU|x}4!1hF`1!f$T{3=c#yJ zT3V_vEfCcKm@g&dsajdMUtc|5SYWtOH6N1qQ%ihg-ae0V;KYVtY$_cH%;~5Jw-s2^ zA{3%?m|QplOxM^LF^)YwV+CK0yqy8?`qC2N1;fI8(860%nSpnBWesu5c!j~}EcTG3 zu(HY%1TSH!onaTQ13}xTM>rq&(vr1Ip^c_>%nlfai>6Q>1HeZi7(PG~gE4pcNPFoy=LK_t%{g88 zXc#BFvHU^Fz=f1(a_yrWB0#;=-MrLd@?iCw2lrPV)Iedb6T<$0m4RLL=ij0g&T9lh zAW3}km8XG&gr|AIj*Scz>C1)c+MpD81balP#FSEQ3cX9=hg!JPn_jrnlYW)Bqz$Ft zsjC?rA1CGoI>npD|1d6gm!Ba2z^kb^FeCP1TD3JpyWGpa2x z;1kCN(38@a*9futY{_=NJ5o+11KBy`!P&`Zkp6O6vD0Fu+G{n4lyhteh#aw%thfck zZZlGpecw&;4uqy{V*wj~$Nh%+Qu^3fkPX0&p4tSaw%Xj;Dq8sNf`HmZHN!&G`ID-5 zS<&!F9<)h>A*MJT1zm_+8R(Hh$N}P(I0C$7PdorLP_MNERNQi(4G&xhon8STC^`-h z+5|uY96p;3(5T-~mBtWb!4nXkI84?^``ryySV-?A%-H-^V}ntL^j=zslUVNJ?KZIJ zQ1eJ6YrPbY1f*&8NF!3!Ng4}EldL&NXt0qcy+Y7W$8vU*u{?+S4l%U}8m6>7!E-bk z1P282nwSjfvvO2%u%&B5g{Z4_I_$}%sn&5@6Q;b-63>x7Ozy{t3_uo8mvfJtR@I4RXrG_9{-f zE8o+m7%%6Vf$3(x@%UP z+Hj;g?@>A3Eb3ubZYdUtNVy1*g{r0O7UG|8QD@HTG#(L*^%ickw0nQ8{#`nHtZi!4 zmz~hMowOR~bmlb%jEZ5vvXB+r&WLOva$rrOr~hSO|1aoeoT%GyrE`DiQv3{25Fdl@ zpb<|80thl7_GkkCShaFF4AzbCOE^!kaksbq>bdOJVX#PW(;32A6jBK$OI^vRa7ke{BQs1E1a)M(hAIS{z zR-;hJ^_vI?x2P946zN_-OJ@l;s3D)}W}SPIBJM&GS}@K*101~qgc5frRCEX^d)u=&fDG1ES z9$z(vl|lZ?cbi4Aej-IQSX4%Aue&LZ3Y2EU+=Un?J%A$DMWYk{p%WgA4fTFPa7+x4 z#icYxCj_L(;A^Xd#`%HMNEOQA5>ranTi`pix)72JUAoSHWWk$yi}#UDCd1T@{QJY) zFc0miFVyv~xO1xcR#ThNqK`zMHCHb_NQihnBj&0Fa~M`xG{U<{wG3R%%`|N+EIhCQ zIRs&oMi&tMz`pEW*-?sS@yT zGR(MCPcP_6oqg0n5A2cJy+$WvvscuR;kiNLAfQ*{Y5oT4$haa7MkKEisf0$(d_XbA;EF^!Do;+d* zFvTT_>$t2Cy^Woy2O6!H^6`?m6+}yvKgW%#T$8}eDuYbBwEjBUS+oK4_c~f|r+vro_wJIBm;W4L!ps=T)F%3sYga+#AfMM7jrW zC0oo1d7@R9UqyBSG=C5k7Hbm-Sdw5Y6Ww7_(6$#a{#C3^fMpoUuOP|jr;ARDu!T@) zvWf5weL2~H4iw8dex*TUhPv&~PNTUNsOS+Wt|pc;BGx^2MA5*J9{I|RqWR`neR`07 z$t&5A$CO#PYE(fW=0xabJ=;U%t%emFRRybQT-m3Ac_{9C+UgqV5w|w#BgG9Cya6x; zSoh!?qA=x?K>?J!#pqtRcW!tBg{}u3CQtHe3WZaM%N3hs-xf0<%#jPjCi36vlgK;$N&g;7 zVwdCkAk|>mr3i zKn^brKG_mrP@(J(m*_?ca^u+#FiPmLXyV)#Ru$W)#GEVzTK`VeXo-@~tJcAp003D@ z)v>T3DH0aiwR#=NF0h+Wf~&MPxMUh;S_Zuu2Pz1QvQ|BvO3KYq&RSxyxKS#G%Q5nV zAYqWZliP-QXXIB6541++RA?@5h~p4IMJWU%7ED~VGg#Y9xuEBC^7EpB!l$8g4>ET8 zAfjGNs8Vg90cNE1AUG<`GlGp6;mt$15(b4$T1Eqqd;(%_K#(|H!XUv=H|QL;K6rc> zUK(Pw1b~B&(Xj~5Lnv)Q?h5{O>rkz*Jffh;pst#zC(ZyF1I#mKZHZD1+k6vs1Lz36 zN8miRodTI5jbAi?9se$>5h92ru3rP2fXfpcrEW4e03EnM3=i;P7nA=;FU2;WSZ?M*2`Y4_hdfRlBsY}oCwe5Cu?{%?3fsPy_Qw!& zie?8z2+HB0jchy@TQ!I}fwrIX+#K7+pU4p9)VKyNI8JNW>&c*}N-ML~wxv|3Fo%4)KR6(ABR zb}Dn$>AlDNyih<^k&t0rs|<*Xg(COBW;pf=x&4}Vs1&Iw{F3KWj$l$+#;F!`rIguj z0YRv6TO%IZ!TU&)&7q11hplM~!VEMisMJk{&}>+^S8tObyDpvZ*g^q-%#Pydnpoe` zFCs;+>6Il8KWU#xX|8dkp=VigmHa*R$uRI4$Ow&mTHqq1umoQG-Jhll^l9_x&Sg6LUg@0_=#nhqLAE#|vK`ej6y#-1zlxVGVkL8J$YfqA)s zpgT+wQA9@}j$o$fFsq_40CMxda>y(umR06jZ6$a3L06>~s7W9C*pnk$cC8Jni-KRk z(gf=$JC}+WcC5jpOJ@^M=cr*%l{kWvGTSV6GXTm>BVVj|tfZj)riC^mJ&%>eZt?k% zDqCR+nK>H`p|WlA1^%*M;TMZj&skkXvYb~G9bo^+megf}fBqMmVvbVXx0pROT1N$A zS&4kigW<5OBd|;A;tjm1z(N<5}>wMD>R&j$;pV z%TDU;@fnX2hOxhZdrP$E#VtQQqRRADIC#-lkDF3Hwk)`HS5MwZs4)$qa8tsU_qgMU ztU9olBz6~`Xxv!c(>(W)!oTa+aCVlst^nj`MEFLt24^)gw@qMWeR4fZb}b zABOm2DrG84G$|kIpOhejQxuaRPEjr&p-MxItcC(ra#$1WLSj+_oaAOds+s414fMX zDqFsAdH6Xb@jJ_kC^P~Bz+v;XI*!ih>XGx&JbzA)*tTci0P)N9C-sz@#}UhSpJ$o7 z`& z%#MQ2A@gyAp|TIN&`|4Y+^XNkr^IYI&;YiMv*50I;_fIvrOA)&_l0^Im5tQ5|lI?Ac; zf&-=^@7&k$YHSHf3Y1Sj80H?Hfnwx>Q!J#H%Svw#uu+`sS14T=ks!W6R~hk@B!&CB z0!bmbQ!v6>Xbhp9%*4bek&8f~e1TFO+!A)(fsQh;_Xd;THiihAq9S7p(L1(e8$45$ z4jWE;Le!=tL0|LDy}(uo0owCM0G`yK2T(&A{EN)V?L}aR&nBbiH3)EOC1kSnqKI=if{}x^@)+nqn|=q{pu;QB$cwQv)9p5X z9`|8mGSuSisErKgzzsLDEA_u2;WONCgR0c1OF4w;;kycUqr;u(uCyu5VgWMnZ`wv! zdNe*c%xMOjIcJ>uEPY-qKFD?nLFtna>rV3l_9Tr~B4-9KVLUym*d8 z$T>@4cPO<({xV?&vFG4V47x~Dm8G;m#5q}c1IT!-R~z7z)5neL4Wn6?fmv5G(Df z2E*l`X8~$X`iS*~(QFysB&pPb>(6ta2$UQOl-+<>LA#$T&XaKb+=Dz zTsFrd)*OA;eWd&w*dmpTqBN(K?EX9uHB+#or)kV}3Lh2{^O?9d(o#xKHM&c|phBiX z3Z}hK5t$Yrkt--lWntDZ1mum*Ls9}M7X`MC6j($E3`D2~Xo^rQ<{=YUW~pKh#H12N zRW_nBvTRXKrYKk-PhkltWBX{LW{55lC(E;hUJ`3T?LZH&qaVvMhWfwG&ddd=LZYtx zh6>6!Eol*|0SjW-ztS{Dv=)?Jkmz3TaU=zQSb0>y4OYM{u9wYDgW{sf4bEnzy%XN- zKytf=iXt@D%sLF=Kmgur-&BN!HLX!0jAueCE1a;9D;+QBOjGcpa`2NYv%29TiI}dD zv+H&mSd}*|EW%M7CxolKVxoQAMkRuEW1)@U=mo=~#9{%+668&RJY*Z@%z5~fk5pok zU1(hdw%b=3WIWfoeoNTgKHyASw>+#&i5a;RX*LD! zrf8d!xXFmA3DYwdB-KDTxEWcQ(=X^&(%~DWDo%qGcH6S_ycpa6z z$CAdq*dZ0(5C8|O&yJQ&kK*&bITDC;4d|ts%;|#$i46sG-bR)><`v;W%Cd2q++O_( zH}z^f9P{bmsv{((LI`!<9+FD#;i7IyRmUJGxH*x0R&55}=)j1tW28D&7idVWADywK z1hiX2IV!kSddLTh+|fQfQMCPH5Lq?Mway~&R`G}yj+0@1i3*97F3dRg8cDhIKU_Hh zgT2-9lv<2{(=^L5Z z(qx`Mp=k;7K~i1bPC@Sh!4Fyz_cW#rK%Mka@TD_=X;PDb{Xc)RTlE3=ZMQq&e=%$yqzt6IMyZsm2TwL_?|T z=^|rHNr+gRD%g*eXYk+-w3(n}3v4Vb^&(Cp0EQ2iSHquqt8vaU0E&NLFgF;h3=C@p z(`ShYya1nUPmuoB3(whqd z9-FP^{R`pfEKF3!oC24U#8~rL$mw&{xpIOkj1|OGszTxb>5KT2mHoFd_9VDYdHe+5 z(2J?un48@B8Z?!;kZL&AJvjw_^^~X?(Z|v~JLtns(&zoSG$8j+blScDq0 zDH{x+CGiAA;Suqy+o1!JNr_6Fw}^gKfFG8LV_t-h?)978Dqc?!*TVv!PU}_0_iJMe zXHO#@mHVyC!(>r0!hQ7|rX?yh7ECM>`LVH?C+!0*i>p{54pyP=ys-hDeS2v~k085Eo2rfU8M%EUjiFdD4Uj)XAPCSiN6eY34RT$DUdVCWXWD9Zh4hi#HuiUc z^7i`N+z&8OaJ2<4y&Lh&F_VA7&43S2DJr?_YBXO6X$e@9W7 z1T^xDL0OnTdB%I}=Q0=W+}vxunvJO}{ToOrz{BB72Y8vnGD0ZaEzd6NFe^&&K6e(c z&%OTfX|gMOenMv036qWU&RB%g)mZEoUDEEMTTN{cflRI&WTlxVP9 zCHeQQx#+MXUP^F{=UzVIW!@kcG~7_hv4t^xu@8) z7W8mLzL-4R8z<28TB|5pa((V)FJ0mmpOucj5v#AqN6sSZ2y}pghT}a*`6eDIBmkB7#!tevPa3{t)tgDgz!sc8?S}A=cx9nknY^8C3nwu#gTp#vdQ0>5d=g~$K$79O95;BP>E*B-n4K`|oL8_# z*m1bD2}e*1Nx_*Iu~qYkmM9U7Vn>8)z*do#f^rlPsUlLahlvR+(U7U<80I1{y?3zJFm1(oRk)(rHSmy&Nm)hyY06w_S5Xlt zP=26xX!Am>6cENrK0B`{X~Qry|36e$3Nq5LEkfV`d-%58ZH^>gg2nz6ma&eABetPP zABG8Vb-L1S+8PDZlt|CFjX=@*uc~3os;gyo7dqLo>uRGk2RoyG$8KU z8>`az%kOqxSnJYw1}v6a=V{#WU=!O6qm5;2S2|=1TiA^6liW{V{W#A(g*)tiuH_kTfD({6fR$FMmLc*AF{%+VeU;-Vevl zfj%^ko8Hb87$CU#{6+kXUEzifvJLx6RK+kzDY*7v<>JB0injG7t8Q*?+AyW3{87fW z`ryIE>Q_K#j)?nTu3UWiz?x8Ge6_Z6vAXuPy~9MH-9oYZ()ok0S1!K(!mz4%TV27s z)o<3ks$lQ_%2#WE@*C?`QBY`4u~29a#DQL8NPe5cy)@g67hBIs_aP~VCx0?095!N3 zM3U!BSu&J(9tcrsM+hzJos6jpQb97`R-}d03x`mRgB&K1RrI`t+He-AVDQfS*k!q< zcjWfQ0{yUlk=kFjZ#s;BEcA#Wz-fiVcxHiJ)43JJe&Yv@94a6C;NVfW429AtsCqzn zTrdkPi>g#NI_=pL;uTE)wst-YMGFWKd}0~_p%jp?nIa%@GeKbl7)1oyYY5xMr-Gk4 zmiPQA1`)S+#YiI6&U5ub3Aynkm_A{MiyHgzA#AJ0nNLI1+7$zUBiWC-n20I?4}X?O zt`a2~yC&8X(A`rR&<;x$UX0+ER|^Hrgd3Y6Gx!y6sL=B&ru>c13^WyWed3eCnSU{- zD4Q$!F|;h^07 zlo{{>fN-J-MB;dL3B(BiOfBnQCHzCeUM3n0FbVX4X=)HR*&TBhVt_hKTo)hNyl_Ma z?bRCv-IaEvU_#hid65nOebl(IRYO zKIz35I^~m4PVA0NZ96{aH9r-yFo-u`!1k-aczaFl1u|E4T&_wU)e(SNG)WVGgd)PG zy!m0nn;|COQiO9T40$SoV*C{6mKHd)J-EqW?vaT}+&Y$?FOnGy17?({Xc#qgwGM9H zL0u3yO$%u7b{B`drHU$Lo}!2-)hIyouL5UpW!c|oQsjn|#Fq-&-@zekZP3BN7yoHr z!U|mvRIguP7hFI4Zn26ph%M);G!Hj za$~|nqH+JHM~8RiF0+k!X_kna2V*JpQE<8m*Hcen#Oy^upIA^i-Jm_lu65k_6)|P- z;eb*ZnK`n6&3_bqJDkRV;;sCcvQ7P_?hRn6WEY8&G)I5IRJt>zpOk0q80p=FjLdB@ zg@_!AnIyyl*0(0R6rA(zhtSgJo0)KKU?Q!pK}~dD&2+?PeuYzzv4MKytE#5hclM*9 zn8&ly)gsCqgWpcQxG0C!Bt>HdXOfYYhvO;6z^Z)^pb(h@BdxhP*KEPw*n?d*vPS(2 zS`Kpc7wss#jRlC-o-mudQFb+3M$K~3iDwWVzlALJpxr?l#D+M?u&V}_T*08r#ajU) z4C;DT^^YJg*RzVq^+_tk^TbRbIz55qF82s?M^JYN>D>pubCR;O|HDdlek{xWq<(jE zzABEPU_lwxuE}_SYcl-t5$?x1z!d{EG=pEO z_!AMa*n=YvZI2BSCcfy50mzKOnBgr$oSLX{SP!^_A7T%xPIw|hMo?n zCO zc)jXIOxiTI@L$2sG{^;ASFE8Tc(Sxzmiy9<5Ot&V5O@MAq5@UawDEK1HgMFObVt&t zjQZFWmb)=>fK+Vf!GWJ@olVgM%8W190!j;wGvkh#sjDs52clt-;z2>Bp|JoMKa=H5 z`FaB=;YrpY?LXW7llLZLjEGDrl>Q?c z0Q%J66X(A2LhdP@BKte-jrp{HzQmFc@sFzbr}l6;1K5us<`h6v;3U=Rlb3s2&z^CO zU`*0r*)vn@JDkc7(tm$>ap@3k5Amri{Oh=6xHHmQ=q%z1%z=ko83m;qk_*V>^KG*5 zS4v_2z@z>?vftwG5%C^sk^?Iviknjr6a-=kYeb2J!iFDEyb!;Fdn=+LG6>M&t1&J6 zv6QGR{B_|BXNiTGnDJ;5j~1q?&ruu^@ab@Lkz=zCuT;v3G^zLkTv^;H(uU4TR9ok` zFSj39rn%fhT^Idf%G^I>0xq+)qsEw1LylB!%nLQlPY{9(YS#1{KZqIxbOjY}3J(dg z>U4D0rN0wyN>s~HfmBt`pY6)~s`w}%TsB!gJxmAzvXDj@ zl~RO=VGPF@1Jx$Zt$5$AU<#sMMaqG6}55q84i+ao(L8LYmnP6$eC{1 zgy2Ki98L$zIK3r~P^gvq^tg?HVRAA2Ap7z{984xG&flOKB1WOKRV^RN82Vxb8MIDh zv}4UqBbf)JAzWPMRYUhc4Vuv;gX8wT*5=fqEFMuFmp?+IF^FP-M6NrtmKc4sbOVag ztE>WDVGV~(HKR;oRU4Z4QivM?Jzm*g6l3Boqg&mo(5$aBdLd&$<0d(DY>~JV>M4Fi zD79~vc~{EzLopm_+L$b~KT*vtY7OCQv^uiP&4*Xyzgq(T|QJ0Yny#Ny?yi6Ymo($83oa zWQrV~a~IyEnATdwt<4GeiJDx=XJ-*ZG&h=_r_I?U^t&S*KdNXqy4;B?LPE4oU`dyc zViWhUqgXT%_;^@5M8wdyXfuz(qW)Nf21m>**p2+4j;GGq(kfT;EO2I3Q6L3?tDppj zs#ca(F%=brK3OgflVb%WY3wNpZ2#E!iKJXENVe7G zwdMQE55PNl*3?h$bHBE-rpcBo_{i|`K*n+?S2{-#Z8#?Go7vsAHnFU+(B>%1BJTj7dHzs&hXDF{>4HYuyzJ zC<$G0<9Tg9$nq9Ua93Z0_y$Nbh6DOMJ&r^C(4leGUb-VqO$=(<1uiNy#rv;ed#tIm z0CiDcfo`;|(tFj*iUZK#2`Xp7jsR`%+Jy}{Dso@U6hJK;S&ZeX?q<$GtL2`Ng3xJ~ zb$8uvoD&5h6mvYzENn+<(6F|v_7u(nl>Atsb*UicEb7G8mn`BryL7~x6Ssla)k8O^ z7JC`|BXFIME+Bpu7>aUKsK&_mU1HG&#MCCW#wE(_)x7)Q$3UBkiSVXe$E!qkRj3rb z^J2N@9HsIH{rsCONHZNsb+4zP>9UWpT;2a^8DZh!Z~ZQ8 zlV?L6Opg%uaO@4X2LZa+pclI|sM=!B-d%z96Ep5e~t#_r44dskLmhQlYl zVDS28r-uvE_ps%#q3ztEsXfGrtaSJtywrmYZt$~hizr2YTQsdAAr5VI$UL^SQ9PED z*TzBHa0((nqO${j8L(V0Rn9ArOmzD51_4g~0Di*_PCwq>$#5_NVs@Y?v#ShD7IJ_0-=H_lzu<}?UrpVHBUB%mDTdvRI= zjhgxa(uBM&WJFWy%@;T++Wl1S&ssT1gsZX zXv4%I>>KWAz~s!uIs%S8@8gS(mv5@caKS91<#&b|LYj5cst0r|?< zfSA#mv}4Q-xSW|gvP{J`QPQ@7V*+APWK%`S9HWV(r^FxPcB-3awL1t}?KIKN26Wpy z&~332tlelKOazI0Ewrd2(>E8Y?59ZK3I)IlR;kzCU%C&EB;dyi_7R`7o4DhWj+EdX zVS!oTYw-=)Xu4B`-B+>OS89$ZGys}Q!p6=Hf=^-2ASm-0lyPfQD&vg8W}|_^jRrz+ zNXAuf4_Q8B6vSX?cghHC@9dZ!rgzxInNOh8j#J+^yUB1S*g;YT6wJ*#s^`}=;*8tQ^NaIu(@VaQ zC-QU!YCnQkceyZD9Ug%LhLk!%D0rrHSChk}O1I%P<}MQ>@QeEpRk+ZEQJ6m>Hc1@0ni_Nqzp!?Sk8Mt#IyAT5u%r8ecI9=?A(g#S|XMZ+)ZIG=OWC# zm8WGe2Uk1q_n^Lylm~~ut}WjGZCSVwU0eSAWAVrmD_>gkEe1zYq;F0l` zg8D}~j}&%J21ajwB3B2!){T4((QtU1Me(GswOG_t3lN%Q{hZ#A2?)2%19MY{h{0%n zB&=z1A?1qKe-AV8W_u_TjBvV<~wtJ-hw3(P~?bF*#of?v2{m>jr`t^hA!Mtw>AIw9YSj7Ji=D{-$ z<`1D!;N5JCisI%vL2^@dIq2{X4JaBECEhWOG@i7~rYJ^>7)~3O0pMOQ3}X@0=S`Ok z!_P8|NOit)7_lnN1tQ*jn3K>|^(G;vFq0#smJcW5qp~f-lEbu%^JDM|W90eDN55nX zZX;rCPC=N)Ktj*L5zx`)#(e)|UG$YW2y%1RRP6nq<`3`I=96VcL{7J||1M;zro`!S zw~}))`GV0j#ytcVHU}xgA!CZL9ncBK5F1lQ=uKRuZNn46(CeFrVo$-S+9V7h*36l} z>3J_5+KQTgwExpXtSqSQJnDH55y$!hLC4KUdEoJ9z^?1IH;(lGQ;q+hq{e^O@1JyU zvOR5zm~BtX;;NrW<(q9$ON|4oa!`kQ4|g~IJwQ%`+UEOKU*3)1zW>|c{oCRD*0A=z z)qLMh->2F8D%yv<;1aYIh?AU} zK(L=~=vllO=aB!qtNMrkrnXb4QM#EYpUK%0p4HA@z~+c{oCVbJkiybO9)Kq&oH8=p5f?P^Txr6gqKn z;8sTznn+5EHy6upUJR2~9}ZJ1@kCX{%ARv3S#`Dk~2-lHcv3Cj}i9z@Fs&WcaYL5U{@(ci$sT~w2TglLp5I0WWhTh(G%7{3K zQJxBwzZ$YFG1#c%#=8?PXaCr4;YesPX;h&QaQ@e6(ENyQfI%cr?b938NeA z9GTW*cIP0kdo!~Wi-ns+NRWLfdAy?rG(@_7cmH4$_M_!5V@$?D9XT0mrLCi=;y;5e z84c$TYD!K(XhQAAv9v}xS8W~rX-SqeebF9XX~6j>45)(yrc2hjcWl)JTg8;MT=Y{8 z*h}z#7vlK)4-ZHFC~0qd9fD!k&!CayfpVYP46HoFFPU$_vp3i4=?~CD-$8CizA2+DN+|CNd1I845Og;6BYqmCWyUNtxqszpigm!c3 z@juMw(ws69m1DT{=*)t7GCjoBm+c=|=_(G&qGbNmzL4cpFcWECe{G)j`yZ@#Nh|eB z){XS8+wt8@a#;}K*KrOykxu?aqw=`)Gv30{_+i&C`|z@I(G^cFo2(Y^T4hai6+P!RF>Z#@B)?S&Hn9!%|jSd zn>>n^1FI29d8d>8D7@ZM+)3QzS za!P~({iPW9@v+XOTx+&j#SN5tGv$O=Nf;yUMxnYFJ5TA|oB=f!`0oWay1Ei>exC-o z>aWFJwIEWBm~pnqj%6qg(AHw&=-}cq-<#z~kemaCPWm}xrlg+R&$T&dtHgD~KTSGI zJsjd~KJA`fB~Du>CQcly1*PTl#b<#8aa*kPsS7ojaPlQ2uq>?9Dcc->e~{I~(>0H- z8g-Vwn16(LDm73digUwEY@keR2aLl$v9mfwq7%ndJo?8n@0dWPNXBiWH=7;zl;fMgD+flynYqXl zAUC}>B2NHq7;|AhRdxU$7r&3jfS!D%UFXl|HMSQmCQKf(FxP;lucb0nGhBHMbPg^S!qY@PS z0|q7i&Ki?^ER)JqVb4GvN!Avfhz?FU;U!FIz#V8rDi)Q!+zdDWm@^j@`Aex@TZ65 zj?Lu3$||#lVVb%p7)j;T%EiT(o%VnAeq8&dJN%E`uP-jPc3xEyq(mchIY@G)Bu)$< zK`!?FX(^zMqYiR=NaEkhSGf9-wVJIK?z6J3#veOxI|KR-HJKF{`YFy?B6t9a@6p0+ zd%zX8RzJhw>PM|?lzT#!$Ajg|bg*n^)|Qd)e%J8*^yRK(zv-a|=&~D?p4WS?wzr-qTf4hBPHjq!$mgeTo;};$ z+WUTEetENsD=K7<@?g_q^NR-B$LTDS=2-y0#DkQ?)SAJ+Gf!)Pd^yH&d&)$ZK9CT% z6>?=)H^;R*5=}kjQ`WQ$+(;`4b)t7xDzp;Hgf~`#l#fdUFpb4l^p(!){1hu)(&y)L8F)IZAz|A)!WG=rCrpQyEla{Sy^UNJNVdrb)1^v*3DUqn zhZ7l?_`|x718in^eS8=~?4$Y9aU8O)Ox0a||36Y)KQx)oeDDp}QeHm8v6W`G-ybY@ z`_1-ptJ@i@FCdLcei^c6R>s-EQM+-9yB?7j?9cdxbTl5Cg7cz(&Un6~6!$>--W?Z2 zGfU3KNWLE>TH-qt#U&t?{)aY&84hqHF(YEyt|w1BNcGHu?l7Uf5H|zcAcD;Bew)D}_Rz6HngJbS-vj90Zcx zlydSG6?+lNc~tBSGz2ZksUm;zIma7%nL6O`wd^~XKH_}Nb)iC1@-EcU8c?+dxGK zDo+tcBNOlgdJY z&x7@=#!x~$2#>(rg!p7GDbP^(DwUF^uB(*T%H{v$7Vd9UG zHOp-FAU1zrSH`%)BdrEQ5LT0(h4SLC8W9%_b)t9IxBEfud?vrf#0f|blAeg>`z}S2 ziB4-qJfD4qW}J)Fne1_E$ErG{TK!4jOl;8C&NPCK5YVE zWwa7P18y5#9J7y`d?16Iz2`>)Ua20$_?JJebhXkU{x4eTV3hp8)U2h;EwE#U%wpV< zgNBX&(bSGv#7=xq0&fQb)}iuBaa@prqgYaDhzvLQI;y}PKVoZqks!}9_0Sur>oEKzFOU9~ws;Ha6tys9v|Ug34=VGA z@YUAu-l^F+3EHv9G*X$rgKS9GYc9CqY6$NVZZg!J&*l(*k!At~b%YE8*u5i?rU0@3 z)7H{|*@?AHK)|S#_gR08W4Sdw?)d>gOtPwSX8jvY!tG>dRLPP-ANP?;{sa1O;KjFn zUKk=L=|!p$n;P1Upfzp~tC@vEpYZ;qX9MWw;^5}NX4|9+?!Q7t+fM{{th0eR%-pCV z*NE5-HhFoqW&Nm$?PXj+YHFLX7nHm$G!A?;kyyOvGYTmp_~3prufgGcAsnZ+3+GY@ zhV|y<(pP*4T48N}^>BYxZo8WT&6MIRl71)F;ZkUg8#}i#_BwJ)6?uhQ6VVG}qw?DD4{ynBiLh9YuIMJQ( zrdDY)$)A(=aY_67g9MX#-MuD4dV#n1c`YeNcV5<~Jhopz=;-t>I?%t3;VEQ5z(DTu zXf?a2Gi4=38G`?Zo%ZQR$9>Gfk*8w^?{4?cnqY^d?E>m#?N*t8RZctjt`&Eq?$)sB z46bJ~_T#pgCml4ieVA^3+{8BfhwuK*Fh+a~!4mNoW@PB&KWI0Y+vrUuB90HNf22W! z>!9u8tSdP8hh!JV{THM0QTjTi*qmYAP))Im-8>J!of5a(GZ1JbQWst_Y%^qBTxKfc*A{~hfNm)eGL@r` zWA`0PpB62{OThv&y@a;i7msd;<<5om8>T=MpK4`YjBhQ1Z~9~ZYf+b^u~0#;%dAHC zqyv7h$V?(r($_Y6z_hGl_*OCM#RNI*p`nM30=b_yY6QIEX6-0er>wzp`%eR6*K!05lmduwCfX1i+b{ zBdkJ`1ewh->YM7AZ`L)Z7D zWafCbMelPJ0mkvz+;F5sHTcyNafo@f)|bms%M)lTxpln2#vSx!ubzaRfv#0?jTTw2;y`LcZZgbsLEqp#R)4-FQ}}S z^$*3ZP*hG9z%2pUXDjf*HG9<>uI^rsA6E|#-8;8YsqL>F?yswdZjM~a=}Rm%fDx0I z@d+=I)tiF>V#0Gnm}4kXBs0sGzoQpk-yNEf-LRfDPPdRodT>D~IK0`q**cYYZ+1X5 z!O;llLK|M&I7B2~U5d_90_6}mG&hy!9b7%hE`kK%698t6t7xSnjq6kOJDZS#N8}1v zbtV1XkQ%QeoVI9<2Ea`a$&1x*`I|Il@avCL&9$;Q1|r2^mVSQn&d~KC`nk;JN2H<4 zM$O@f3cF-8r8+%LRhQPOOBNCkBdd4q;LepQyW}o>M1|cmexnD?p**3l+iU0F)715N zlvnHQp^s0bvWNCSA>p?vtW$+SL3kN;xd?lu7v=sD6!sXSpH*uQNc&T1cSGxss7fVM|ghOpWVt`Tp0R3Og|*c$~_k4Ux{BSvc3G!RU1c{#4jy%fwg8A4*{t^jntTp?0Gx>fn9zJ-^pB-j9)y2XS4S%kty>U^%?Gy+1fPTy8Bh zQ||6FOFp2HsY?yGy75{nN>T!S-&n%?8oHoXUGNiN&)rUemLq2tvdx8z>Ob8M{Ck5? znLJ2FQiNRXbH90*Oq`zA(7o=M`@_e=IH1Tgly8+%$-r*!-|b*8`rgbr7!Eubt`3vk zXA^gjb5@hv7U41##$u#O6vDzzWs5+2kG8j>LdDv?L&q8#sYnduTcr*FoB?=G#tj>) zoHEvzon$On|{yP3FZQlU^eZEKXwxPydc7AIj2QapZGKt$9{zC;Nm=mL%Dq!!pF;vqsM z9RMnt05XI|FXENYW@iUUI3b2z3t6WBbD|?KnYd`GVWXDdm%k%oqPSdFYaZ|r^ELQBRIR%nIh1 z$t`E;B+~*pTu-zRh`*{f_y+3JM{9yqxn^sD(68`6YE2%6KV26T#jW=Ka>O6&xXmD( z9ujrz3RUc+-5fOwVa&(4o-JE$4mzR^N+N1#a^F4K+S`&4xtQeWB3JF+`Chb|k z^|bv2(~>NZdCXO(!dL(pAwvp4iGIN4K38NjXTVI?>-9}ZB~2{hAQtXnLM}$ObkgTp zM&w|^p)pJRTIY}g--{z zA4%`Wi7h8x9$05E$=K@cWd0aAa&h+r9A((ygDpg3jdv3BjGbJ0Byuxf)-aoxOa)c3 zdYBjum!OW%cwQ4Wl$^4bC5S$F29qbV<*@A%AN6oG78}+j*5p|qf4L-aRMHdUu`<^v zK9?6cSLq4BlJ!YLlku~$f|)#=zTQ-WRqc_W`|`YJRuI-cb=HO+AH}(aLR>Ct6m&OG z$AobD{Awkjh>T6qOgEr!%FqIB0JQMCq;N7wK16(lLRVDqF}sF3G3H1h&+nTGbC{6h zh$rQ7zuU!Ob?|b?-<*7^F4B4Q0AGyfB8B~ga4t;CVrf=H63hDZ!AwR%E6BNaHQ{6} zJM@amvo{RBXqxjAM0?^6=%+YEkljFaHqVBg>uYOe9*NKBc-Er~(_N_R%bGvc9H?>C z;6<(1m#dq%yL}uXPT{$fc9-E)EZ_Ya13g21^f~#ct$kI9%3{nf#N-?-;=-9Ovxt}U zK+liq-QP&{i-&V%o)YiVG&Td!=7fn7uL3Q~!sUr(P~{^5+}6_$r1+>^2KJO-)bnx% zqQRHzg;!-NVMAEq6N?n7GKT3Hj#o4s$(fw5o3oz50MLP-gjmK&2t5UH8*19oxYKR( zGgnsX_g4wMp)0=6UE?u@%C1VfbL=IkVOB}jlger(`3=C%R+=yjah6JY zg&{aUQy^4l^&?&m*N_C+v*_N2EPhO<^P4!SEc40VeS&9#5+<&Bc@wUAfqy_UnaVu* zjJV^saEix02^9YttR+5HN_t-Aqzyp#1j=)hMI)tfGXEjG9A`+=zJ8y70Bdh3Cc|;(gqJ|T)j`NIP?1U+p7ln` zD6~BlLW~xN79Ul&Z~_-b1eUy{`?S|NaEo-kKY6*g_3W8UaSW)tyK9pFVG`Q#UG)-B zE6^`UX`F%3x$B@8;Ea!|m4oaadaWe*r(kx~>Z~=8Q*Kk%{lskRHpY~`1pJ^GOmdKk zmhyvN#;C%AWfpNUxt(^fxTN`nIEd^C6$m!C+|Cz2!nwA$rV3!Y=+``cPly*4ri%~g?$!`RZp+(Yz1U#|E?H&&>#^ZkVR~DJ8 zblS+0;OxHKm#1=&)b?5klQXN7Wt%N!UAR6EL_T)wFH;PyYmf3^Uoigx+&~L6;2}5p zU2{Op&TPjK;)J*BiBFrY^r+u|djlPDTXh4|5xwG6-SDSbKGkFEE~tj0xh~7U@8u@? z+5sgMjlohOf&L{#2UX>2@u`Q6ToI9o&pJ+#|Ln(VrFA+lXxp~20w zJ3Y+O20gtC_qP_-p5lLJg~E7o+CNL#z3g?r*xq~fxbb}J@%K9~=#H&;%LJhL4kwR! zjc!BdR<^AwG7d7qQ{3yse4zZ@?Vrm}*fn-fQXm)BBJo>AXaSy-F>fJ$=}qPDpfbc8 z>N$w!{KEq4j2aEo&|I_nJG5u0EOJAQ9&!b%S2^JN@>aWzv`x(9E+Rz>d!ZXCr9K2| z%v3zRgQ9XQ6Dy&E6@5TAQzV4a*!kt`qd`MRH2~58481` zMUi;*uD=v@9bpz2(>=o0ImaJ5ECLxZSmZX9#1pxt5bDeX(yF+*o2PiW3eN4^I?%C2 zKuiM_TYm_hkqy8C_H1-MbJpx6EOwm_)C_VNO3?3^857zY_f+tLf4GDy#-)+q1sAL8 z=;f7MRVo7X0J-E)W3CcqDXpxd!ozmwo!@rI_RfdRfm~I{ zgL?>5`iXL?CR$0}@L0Z-23y}4m9~Ho(31G8_IpNs&hI?*nVz&1sm^F->!u(amsy6Z;JGVpZ*ti~wt%f|d@8qZo8E*M zKH@yJ5ITTq-RFp4=G#tz2$;|hR!AM!&ktM98vy=9m%5wo4L~@nrB+@E@pvZM*sp~8 zkmz*LhLKJw>t8D@E16v;3I(79d;p~v3Jcx4c0I=#2EwF>jD_-iC#|hU+G;=~kNOedujYlQ#*YtgEea#J<9n6XX^n!@rC9~hA&01w1 z{vvQv(n1>YL9DabwhDQD!*T=DB7VV=Sm-?@B?S_zCVj;b8ITOL$X!U0Hi3Z_u)ep>oV$ zW#@^g-J;WtO!YG@1(l@htJQIDm(b5%FA_3Czfeq6}fQ7^=b@=nzvd%sq36&EK=j7vprifghrvm>A&X&G>+@lTh3UD9sEd zgJp({ZIB#;D4&M~N2T%J3aMCAN{g%o{Tj4ZX~&a#;sGqEqWZXO0z+@hn-gYzJ?Z9n za#P6LPoUeqB{Ne#V7PIJLn&9VL_8WamM2(D4+B#zlTQeyjPwTA#OD~% zAkyfm?75mKn>CsD@Rw9`-dZ*&X-J|mKJeCgs>#R(*`v}ZSEW0;{B7mKQ5I6(P0p<$ zEi_K1;kfXZz(<7=IOsTro#?y1(HRf$L$yP#+m`v4>L>TNB3~oLYURZi+a}>gUn&q| zWzx0J2#cU|J?=5CDC!u?jPRBPMeV4M5lzIb+R(%ZM`QF$HEK^r4G{8r6B?m&yz&58 zg?{{F{-{`1 z=JrEp6RZZ<4}rVd0Du=mgLU-{4iO=G>}IvKY-odxU=9oIMAIl%QSsx|rgaVx;$M6? z6IS@X4s+r?&UYhfvKn_-lT@MkGP+p@99DXP<5E!7W1s4USPmClD4`Po3Jl0Qn@Cv> zc*nUdw<784ERW~JWKF0vh~|0QgKP!4g(oeiQMC=yH@AJzorG+v>P6iI^^*+L+s(u5>q41AkPj9ory-vuE>ba87F z?Hc_Uv$~!@x#0RHxI;Pu5fx3pFb2hiY4;_j;DW7O^l%@O18X2=@*f ziD-Ry*q2jC(g0>j>%hIu-X;N^vX|(ai3&oeUDj^I{Dw0anBpd$7IQ?-ES&EuY6TUo zIH*^;#4@xl6++(m{K*Tba_AR-$&-Hy8;`MKAm4p?A}~Zh zp#dCEPr1FCcb}_10d!ZA%5}U-EXuu7^lnx)hW1TfXg()5n!8>8rWe-V4|F!0_vi{Q zQLveAlq$c4$j~hL7PlAg{cwYAQZpwMm&p;T)kp6>Y?qXk4p6JQj2O9p<#rJED?iom9@MM*c6W^yi1z@kOAnwkE84)y_lRVxyHgHyixyk30dE*hwvcI%+jhQ7 z6tzajym}}N);!d%wEfoZ3=xf4-gi?=gM2#Qbc>q*SYV$>XTej9^Oz$E1)Tz}Lg2t& z6#)1DxA(2xZ5>CppQ&GQh8h`AOn{c-%*rUDG>SyonwSztq8w*YG8RaHBrF18Z~#)w zIQO^Tz3b84=LJBr{Fo%UcPye$KdP&%s;jH3s()#t3y_4(uJu5U7Z7o>yB#0Ha|s00 z;^lVK&O-467Jm=-BTF2OcJPPO&!1+k{OxG6!_S{wBAyClwDb8VlWC8)WjlA+)OJRU zfR?y3ipiqAQ=SfFXQ+~7;Yx}1xU}esYBPTMsRTWrB?I7R!AOakoH%QB?KyEhNIZg< zI1J^T1H{nsB+;Tz63iSqA17$*#R)tLnYyq<5bScpA1KnHt1gHDkdO3ifyYp#^6ro~ z=e(>cs!5X|_g)=5>HOZ*2ki#`h5ZNu@EGTw8Z#~}vxf;aNo7~*K_D-Nr;BrNlKKt?(p1)wSE+;p&Z6<}qfNQ0`c#fD(ey#v zqn?CCgK|HF+WES|%0+Ts(+Tk|8N^B!?w;eFwY)9d%y}v~Q_$>Baq6HpE#KJ0^y#EE zwV?i?R+ZCQ6hYx0KQzCwA>@)6JOBea7>XGp`~iCe!$*c~?VIIb>+wYk2bk8aap%^c zbL%?0_0`>5d$%rx=S%hroq{QI;w7v-jfaC#zx4)Jec%y=c2O_XZDV#_9Q6ZQw1rcS zOvok;BocoyNtGsDdU9yNo97R`_3Ce^45azVNcKBb1KI%e0YAVzHre|Dfm zAyAdqOe6{OyDkfMHEfRXAP8KP#S$7@MP)ed@QMX0zY zYaoVY8iNQ#wug}t71b(-l2=?)2RG3{1JG$%sF-S^*5eSb*$kvNK`n(_VD-4fTYnl} zi%Pou<}&D{71uqn<|UkL7I$Z4>=E@hnd=|uoaHMm3MVVl7>U~i_zJbV(pMB!YiZpU z$Fg@VI{g}Y_in*GaeOUYjr~;|nxI10CIIs272FQJ?%sZ2tzM&%-W}xI!5`l} z*v21~$9k=etJ}Sg=y3-r$ooB0m;64bXFFHHrK+OH`%RR$D)@DIVL8W)&1Gq@ib_k2 z2jfBl9e7oq2@KIHq25Q+*$dFwa@Rzvgn5GYBB$ESiKo-R`(_*v&a-}rKfT!CmA!;@(!Uq0Q&IsSxoAYx>P%C`@{>%KuU zvhOcWU$?Stb3feh>8wYNWbEc0#pVguTBtcxWL3c^-*k#XR3J+n2-~ggTY2~7{Icxu znJ|-~spT(&9)5QZ^ZU6U2Mz2cYDI;^oca9g$GzVya96NpU|9-VY9a8!NlsOhqG=M0 z=Eb6N(G#`spk$(nL66bObm-7-xBt-u9T}yQ#nvK;o_XuA5gX>2);KMa|4Nbww-z;`9QEkxo;4WnkT_=lUVereRFUt`2l)?h#~ZLKlwSRCJp18T zJMw=g5&`JHROsBk{ewsgPd{Uti1mG?T^G95KD94IOCUaXo00LI;l(J&u`EZd?fyOc z0q0^dAg1;~GyuAe6fP{*<7QRB&|0CF)W*BA0zEbl7>5 z=+Mq(BdEnijRr-#(MMLH3he(rN|b45b|!8GmV`kMgyPCu&1ae{+AT0Ht!pXoT1$%@ zUZk2;TCIlnXbDX$8^97ao2GK*gK$rzDf*~Ov0odd=w_G ziXQ8hHGCgEn~%@G5_@9O8+vi|lXW*(37=!#UqT;^7Pzoql}wEmDG}X|gW^?Q3niaq zKF9`#GIiZun?~wRFE#SBtS>&bXJz3bE0-j=^M} zyYR0H?ed^63WM>E5=sj0`ANa{@aAzN2y2kzc#pdGz6T({k(rM+B2uHx*Vymr%_ukB zolViwH?zzB`2xRwxP;AD*JJax>-VBScgOR~3ApfbgiC8-XxIui-~7fffbbrPWR!q$ zE+t77$f6@f3QgjsvKmd1$OwM+1Xq-2Tpda@Kc=PY6wf;<>tDeiW!Ap}v!o!v*989` z;H7+->^wDm35+zdWnm~`&`_dGtSxy(gKmB?KUu*bqwf{Egvd3>r^2T|&|)^K!QCC= zPWE!(7vFu~$j|za1tg8Ac&X%*(9#S)Om>=1zF5nCjtwaV8ki>N*>VdY!J4 zj2iWnnw63o-->_)w&j8j|BC!{kZ=D=Ywa+Us-%N8kY|CF&! zTAH>$UBKARH(a}OoHIZV8={Zw@u)3Ltg+uB;<;^0WWtJ9ja5E>gT(=j$Za4fDP9Op zPfkY1c#zz;?J7)q0xSUKjUMC9q;?(`iz9wZroy?X;;hu&cJijdsY=_Y=hLJ9`L-sz z#9;$!3Nm6bo^D%Ep#Mcc=WhC~`#nV@>7T>le@d&2N3nz_&IN|YziR}K0`{U-*$6<3 z3`)61ztpl#&^&C;^_&xiO<2bbzEYtHJ0{gI9`VqdEKqF^m{Dk=%UWeSxI93Jp*N7uLm4 zK)9Jbfe?W^=~njr0-E>*A`2uwl=%Uun%i^_z2V7; zO9K-s*iKpeu)H*wq1Xmut59w(Z%AaLqKT5kDyS@i ztAK%M#cqnKxEXjoyA)KrRLqkAA3nEAneUp9;~r_UQf8GZM}Iqff-GyO`!Jng&{BWA znw=A0f0s4&6G?SF3Z^~3Eput`>ZkEiOQ^C;e`X!qWyms|XFtd==8cEbR5?2v|we zK(y{5I;fxNVK(t;*p#HH(8G{hN)D=?%8Ur15J~Ey!$?Kl#Y89-{x&{f@Izs{(fG6x z3dlxeP@Z4!Y)((_%8l7dFwh+YZvzBARsnz@t=9uXC~74bq0 zfqpIApa7ZCe&wkU>Gg)kXE-%73&$1?YOP#Sd%Tp?9!M?#gA!*R&~0dlI0@&i%@#6@ zv^F7yP}v2D)&t%W*SOf0tLUO5aRo|TfL04mx4#Ks)T`xM#O5zvVk?nAsyH@<%~RL?XTW`w1rAKG5J+-ZtbZ0>a*98i#4 zGuoInu>>+TUaC^2B&uYpwaEQzs7NjHqeedLt{bKiKyX<9F?3F!7DRZeb4oEycx8ps z3CP<~#5X%*VcTW}=X`5iv&~u;zm`o_X6IK3?qb@ID`Jf4q?uqwgB{JiCY})^n)N3A zb3>?eaW!r3*dx)zwz@>F*_|*vrnz|=tH9!<V!1);g zZg+Vcs0^D>8}%|>moKm_qTf?gG>{(t!b;Y$*0^WSbGNF7%qLuIw{}8qzD9x z8gp1k5g0P5v3+XjPn3XYk&Jz)ovX1mY%=0|QZULB6NfERXxOwIbE6fkYtGOBhO~(( z-^ze$#@F;l!nz5%P9v029jbv73@C$YZvsMhh6_}H$5S_AmTn(v+AytQb~+BG1=0Z~ zVG7SvI>Csi{3<~Z6dYHPx#kQZ$UsJ-8LKfi-g;J~O1&NnU|eEXTKxRFnvrHu)4u(N z-2eX1IlPmOnqP9LX6rTO%o-+onY+V>OuVUO&Z_@Q@lr=K`ZJ~8aqk@;E6bKDiT}uo z*PYvNR+kqI((K~t7Ft!nz^=_2*eu;r6Bf`tlv`-PZ^HKfkcwK0MN^w7q2v6i>&8S~ zQcgDrlt$Zd@O{S#nkQYDu8?x|6WFc{XWWlI%tRbbpRUv?2;5U-mkY|Af{b!(BKWTw zZp|{5Xz^8I;$b(PqwJye^K-9t=Hj}uBh0Rr zK*HBGFbDKh_WFA50OVb7%#Rx9Wd2zdD(4OsA53$lpLY>S8L|t_8AnYm%OJxKA$cI?|m7Xho(G{-Pp*o>;O4# z^|fsH1gmQR@}c2RcmVkmE(n`Gm5_oy{1_3DJw__Aw|tcj?Z@;8IFc}5M$R#%5S*2y z2gx^yA!o?=4X0Nrvz*k@YTSQ|v8-G;?z&PSB=QhgdKPs)x2$> zwC3lFQ?i~&7M1$k!BDDsIk4lU%EISWwHmdV$-@$PQjvUe6F( zwvW{*imgp_w|N(y0u;z^_{aj3MeVvLd)bbEs@M0#mg7>Otd3#CjkK@g_s_)!pG*wT zFh~lT5jH6y{d$Py6Je$XzqwVbh6-3xu{h=(8xz;10Bmv0_m>$Y%iXWdY(d=Jw`Jl~ z*X1Mq?su|H)2M5AVVxu*XN#!2mGfDXq?7O`7|W5yr;n*RD7p>4L_0$d$uB^HWNye; z3KtDNVnL_OkROfFNtF{_i-G%ZJ4u6%#5`=?zjtw8aCmR~qW=9c!D7;|g3FB!jJP~C zE;Y+Kur~3Q;(UJHfn9Idx{q0q&fa{Q9g+;U^q&l`Lfz%dQ1`Z>y3Fq1+dljJJ}&F4 zCsJMS*|q#zO3up>8WPJCj$KSD^GCb{yOrJgi)NK+%qe3g&0WbHk0rppt&V&FS9C68g=2lUq>mxxu^f%-uqXNf^5^826gyaa{9^>V%(U93MT@8%K;uoqI1~2Ve zx-4mjV3mA6=vnS80dPHgXyQZaan(&p7d^1qNnK7G$ix^NN^NyuJ}Vi73N_t5pqhim z{R9fzTti=es#>Qmwq}ip9$)l!3!~YKp^Ut^h($m6PjwUH~Fmc*SWB z08gR1{?B`#iKy`Y_5H8>rJ1Vv;{1t(2XFx`yLn12SM2Q)?O$s6kh!Z{hyOAqGw2X( zl3F&MImBVj|Jskd=fqqOTR!|BwLEeBIp{W2+AsycRSbw@n^v)aiRF0PxfGr!J% zgQw!_6-i=?P90PRWn-%YuAq!n_RKJ&kU!5K0suXDv|W5QTnB(lM@+Ma`D>~7;H~%~ z5W25yw*LrbVrVs*WKgZXc1bU(Jgpo#FL%8X*QtMwm&^0^gWDOBcW?3KbG(5t#CvWq zLBfiN>mlF@*+ku(42yWQ9w+1$Eagd=0#R_M^tQiV6SK-x%|){}diP-lW(HA?t+DG= z<)n3u7xmt7edDROa1HAxGo5-|Vvl~(=6y;2qG6t#9(@|c&)KW5m9CCzP#P`6A?R!= zjA|xGa_%w*%X(P2ulz_`E_g-~;=zQT^j6~w>JT`j2#4qWs=nukz(Gt3@pFCT%jUb5 z7BRTfI)c?-11f<+W{P7Qwbr-#i@|8>uV}P@4|%hlc1iX|4_(9?auLu5QqPXye$=xe z{J{zbKpw0lYvYpFs`w_y%<*2tbPobsT1eTB(bJywJRDkOQ98Ou7<+GHaKSUe=)|{NsL(6l)&aOVa+BjMd{fTS~%*^jng`l~^9(+|6%hR`P z`SpmXux~%a)FXNr{vcH24_S`QvTW5gtC!Pq#0t?#Xo*s{x#XvZ5f^h@tuVwThQ$X)qiiI=EY zViJ^b@dn|On7vrY9JB;vxorolwp@oZsU$wSq0y(=W@i&OA4$`iTMZ0zlOZx@wSBn% z)GskfbWzs%)kAr|-A0h?_ML6nypQ{Kl4xzWy5CPn@?Mm$mZ*@wy-T_QQ25SGM@EO3 zn2ml{+jq*Hu)$o2hEx6`z?k@cbG zlDi@!jQ=Ho+|szaZ~fhhXMGSbhb?VT%s1i7<(mL~+pz;Qz^99b^X?Z~_A+;c3@e-!t$)6w33<@aq6cX_L(4PR z;Q7UH@`x`m&8E}wdtXs_F9`l+N162W0zFapt|Wyoov)c2OMv0fd@x1;>JA4BW>~%G z&(H2}Klq`&y@#v0C)4x6aE7qOyIWbEF^cm)5gT>t!rj{6c`|C(`DK;fB4-e@ONr|A~q=bAb(l;?Fa2oU`Ts_|5wTA>Yz=J{WxzICXU-u%6RyaR!-l^ z8P(M~+*_`F48N7%EgZ$oJX~|!)xrI+TMdsQ?fjHJR%@Tj)fNauIjp~CZ5+}X`(C0V z!&pu7Yuy%Kko1Gh3zh@yo)1G#MYViV=4(6bHEA2~j1{DP>AUsG&m<&#O*e;f7?TYF z?YS5H@KxU84Gs1o@}o)`?D~+=<$7eJ1^<;CR$ZxY=Gc@;b@*KF3L+?aqkvKZ5aRND z42u>JjUOMSlJB8#;8;7}?&BTPKS;*v+Q||n5qFi;B4+U*CKxY=<)rKA((HOk*j+fd zSah0h%8dIe2jAK4MjD$|ifD`YFSnJcJb}_ujJ~sDd?}c$s>#G1Y29eD1Sv|09}uXA z1M|m;nX68yNz;p~QK6WEHpe-MYHAq7xy=nY-*Ed9OIqCGs{qtsv}Tu&l48gQZ0m%K zN+#n>xUkqmZJ8>hWq(CXBF-ta5E|&Ija>c7v)Ck80`%qKUy$ zp;g&=E+$&vjR79F-8f-=I}kbK0sL}UjRDh7zR?w#Gaj~irt z3j%-RfE%YTq(_2KYoFpPN{)2-+aA7s2U|0iIh_0|@+M-P_4L>JTw6DVM2U`bNefp` zDyc29pmAy(Urk4d1coX+gxlB4;T%~LV;u6mzP>rK*%l_Hj!6yk^#f*h`tn;hf4MEP!S<%XQL>ut4~xJt7|k!9PLBpu@>Z1(&g zwERqpX?KwMz7g(aId0o$*DTkyQ<#-PEc3b4I8?AP8(cm_$fbW_OEXlqtu`1-bqFCH zC_*?AAUora5H*AkxJ8^?mFg({;{G2hm*N_puV3L7GbyB7M6obE~3J0E3&k78|BqFV1oG%p>+-8+rq z=%+m1KK|#N1L9EQR}0G9h7YqUeT7|?|U{{*?kSeA^g33w0eDyM|X3YQVqJKV`U(;_ZG)B+i@7(j{`-=3odRtiDv`%NbE3&YoCt_@9yFJY&z&>e@4OxiD!?k;YtZt)6ry<<7LXMe`6zG zNG1%tLm9?k&SeaOx4Y)ZR)HUki?>vb$7hUYL=<}gY&eK$+%j^=0d!a%pg=K@ezusj z9i9x4pu;{=or_n?3BQlwcruy+FO1J+9~cNZ==$ebe|Eayi!+`?1CL`dgjf2MhsyxH z#JYMi3&fmWj;0GhQh%y7`Wh9}6c%Xdk$&GBLm;fJnIp#tdJm7AHZSccH((Rm+hiDiBY*4$Os zJlN2T7$7n2(mPql;Fv@g={0=7%hCY?Aod0%v6~$J%dyb>Lt}w^$HI~cpE#f;ej=ly z2w2(QZ@77`8<6Bdz(8@BxVf+RFfSHLyp{Eo1>HMI6p)=ueic;1bFzcciJ4G&l#l>?&JE2;E#x3jDumUnucBB;9^|x1XA^oC zY-*zmI3N;LQ$m2&0bhs#Mom)AhVdxIQLZh}pJt^Ye;cuoubcqU*yy;MeKoxz@v?sG zY20Tz7*aFgP4Gof;67(4lKDAjT+lcx5{F2ea1$*KkG4vd=#bM=aJt!3GCuso1GXtd zX2atloTnjZP9Q(f6tGWDIf)#;7K*%F(C`#64)i6Wno0CWa477jG6#LgwHEr+-4v5i zr29-LyEtoLAIW}@1PCd6t_9+-41|cH4hu@51vu&YN(Dt7ybXB?=3sC}C*T$8^jly6 zS`5wrj61(Pr2qb-=>&_B10;d5{YPAP?|MKv$-9PU$JjDxV5c%5ylMqt8~e+Lc=7~s zbf+dIWW4?Tc7aLIN$nVM;yaXXc)LI>?zo&_`o80Gp($)u{B}6J5FKadN&h@oKntUG z8>J8x6^Pp6XoXXH>5~T{I4-ce@lwM91x|_u)p`bdTGNU`-7q0k3V#D}{{^R<`RG{L zQSnHBNp&z4M77D|vQn<99*=phJ8YetfKiWd^8&?_S?qh7079Or?Q~Gw)3?xg`%P~( z?Mckn%!2itqdkzFwCTydRChu3C7Y00{t) z7Gx)F|8##>v_zC&AleIWg2*U5T#e~Ffcyz@(3#?F8x3 zCT>@H`dr9+D|`M@{t%3m>Lysj=%hLf#mxdL0-Y9_$OMqNci!uX8QTC1YtVOVHqW{D zx8WB9A{G0~j@8;IcCRfiaLQ=6Pz4VnpgmG)$ODzQY>b^sq1-uXr#w#VNM|k4IMRM0 zP-zw<>;-HjbucHB6*9wG8p2-JJ2@fi%A#*^=J2S3MmhfE-{g@)w0xo)<9KE#dy z>PP?3vbBPp=fZMs%%o593!u5cnT`MwmF0n3HE3j2rL6sD|GBe~&`18XT&z?T}5-+HR zn1tyOU!@d9Dic;Tc9C;}%V80IMb?D1(J9wB2+{CD-^ZhfLPE=cSkhR@?`HNja?Kz@CD*84;H6K+1jiZL-GC?2zBV?Lhq06p zrWe<8c?=e@IAW3#l<{t9x5c8-M4KQDqU9(*q2K`gz5ZYhJ+z-6s~tqdY@8)PL2w49 ziQ1BWC2L(yXGf#F^`;Cvk7H0a^AVy>WYAU6hEb$8T89EzJC3Sjl)+cn0k?uo1}io- z5Uq8Jq1N>DRPYuV^=7uW|JzK^eFa5T0uvKV;~E|lKj&*`0%xH&qNIvo*^dcTX+2!4 zsccEf1V5!7f&~J?>suE#4GBa5jX+;TCQ7LD%1Z5Xti!c%PgsQimGy)3P% z5`)g;FGr)au^(u1)xSg~a{k{BOE8KIk?E&ss;fk-6{u2Ly$8->^k@k4r5+~F@x&}7*mBppT#;39qP()aeF`r*98YK&fsXixWJ8r zF0=kIqFn&JV5uy(u7XSXOvy@QL|YMGpl^#R!ZdK8rm|%8CE5UiEm)^rK7si%;Q-Z^ zTyjzizmj28+mRvOnCGJCmCCJ4Q4=&5d;G>GEU?ga8D|Gf>HNqK+GjYH%0TH{X2bFH zvN{>ESPEd1;=7zq0?hCv4iTUnFIxOyeWh!$Y@52or4x|2W0^%SRTe?vj&aPkYGHvX zoz@XW(@Ur$?Z~3E7VEj0)ZS{fvR9FOaArmtaNAOj#?K!7H&G>3HCVZw>{7=G)%Emm2 z8rR{xa5aN$QHtHl8EWI5h%O#rD99w4D;W!HQ=C?}%fkZ<$eJb5=>&H- z)Y|b6c5J@GF4lxs`;Q+U?DpVzDBC+0!>(qv>KP6i{ehmCDFH6U8wcN_JMd<&XZY;( z@IiA+eJZ)7VsFwpaAoZizLX=B7Kb-CsjnryDqqF5DB$55r^Ln4uromBg&c&^nNKCv zWY*?IUeXJGwQd)P6BEMS2Q~}MCP@T!5jUR02y&zOs2E)5D{C50BMxh6WzuB<*e5`j zEP^jLm7ya6(HD)ixR+t*wCW(<;4)qHH(F;!Og@-!K>NSZI|zE^*?|VajNeW5gy(+CvF>Bk)!4=o&mMOcjtMRVvuD znpkB!_S>mn$jL;_;TvciImO5vmj3+h=z_-%u7~d(&>3=kMK3-a3b1fuZBg45RSp#h zCrYZU0WBy$J0HHY131ibw6dY6eCJ@ckoS2DYp8RG^yk75eSwz+9eO81n-DNUTO!!M zt-k~x6oKtlXUfA>r=uUR33l#ho#|HA>1g|GZ};HgHxFM>&|y}nX4AnB%w3yxd-6b| z*q`pQEbsPlM~2($>US6n$=*rZb9zRxQGL_!Ikv03igZ2vMs^w}{LBOxoM3Z*FQf2pKV87kBeLXqMw1-m+YbL2`mr z`$1NPWoU9anjxA}^o&Q(zkap%>?MHX*~L&`)l(Y7r2}63l`aJcr=z^q_{Dd_=3<7+ zM}Hf(-az%OHRMV;RIIXvjBso~B5bJ*EUy~DoU#ZPDjI>(*2T7oBFftzloSmnNbl1V zcL`wR_n+O^;P12H`33Kt<-ss5*F6-BuGdFbv;IZKYJPozOXL$c>%qyNyIvn#G%Q)z zdh#hx`w{6qFiCk?Zjja-(abj#0J1_XDO1x>y21e-(UV<S#M@D)IKB3Fx8Y2<;1 z+s97m04jNTGV-x7J>Up^ck6hOd}J9WicAW4-d^D&eJoloQ1S8t!q?;8Nw)@j%NDS; z@R6BrSh(O-5DQwjsMchMO4&%7g)J*P#DB2mzma$ZrZzk|$8BCujrY0rZ_KxU^P9IibiV3-zbI@xJyiE#Q(OEWmh9W=`` zp5)bH{tnN019`Q5UuxgZCgK9*-G{=$2(Ihf*VRM3(joLzQJes5moRN2V?1x|;M6$Z zK}KtyMSDDhp^P)}GUzhUIeqwwBHlPf-mhSX9-U4wSVzvuxDy1c!%nDptN}x7du4-z z%KB)s5FzT?8M8M`XS`&D*1(%-yP|Et_e_C-AIh+E4(Ah#h19vT)C%d5z>RdL_4U)e zrw0KQVOZA1H641$+QaT?H~ZaZFwD%dFTUvh$5=G)+Vy^6<4e*dLxTg!&q)mC&wn%@dt0-D~TH$g{F;f`5AWMNlO3hVdE z^%c6W@kz~Ni;D}qz>VEGRusk|+&B%2OV4$7HC>z!>>{`dZz9l(@}WZ%2-tE8P}weuwWBBkHJOY8MuTe1M5G-vhhrCossuEw;Fp5;P01yz|)J*x_b zpMg4_Jb$tGFm6yXhWQTT*f4-$c-g=m7=-MoY`al$Bj6zgJA^3*Ve}Gv@(Lp4U2yBD z96U&K(#K8JcFrh>krZF+3u$vp+!L>F-S(5kS54u?e;ahoF21JB)E z43D{GLEwl@#6lm263W)Rd>!(xT~MxGJ*0~h#iMZzEpu$y+lS+>XOi1(=Jl^UllGq1v6Z&by%&y z+sH%Ai{pzI2dH4RKE1VZ3#Or>9rhD){2SkZ=8XUBkk(RY;7==*J7)s)zu zT#HwvEY~Q1yFQd1?k-^qsG*~xrq0@7Bfu!mlVYNGLWz*B4~IHObzt`C;IzFG4%Cxs z27u#k%T|h+*7a-zMQeQ^xVl&}0%2S73IUCpzIQ7ongQ{8B?;X`Jqkof48LlpARxuc zE8+3#)J4}vk-&xh*k8&g~>W!8=}dSc9K(=u2xIx$uw3iGQF~yf6b2iJngPatR)&>FhgE1>UITeGa(me>}^df8!R zUGc(RcvlpkAvfe(ND0nE6zfDx#7+=C!eX2zVK2cQEokHzehy;B&lE{)3iCzMGf~A( zqi;_HLGsKdbrp(C98KDP!T3n!Q3uNjV_l>>o;)_ag<1M7D zK9)(&#x^D2i2xp}@ZGMPw8OQjNsd41gRCo2CM^o8Wuz-TOjWc`Lfp>~{S&laL2N?w ztC+B(a8d{8FwgXQP_lZgtq4v*Bq7r7WAL$hd{3bY9h3~eODn1vZu#zrZyXvf(J;)G z?KPe21pz`c3MLpAiV=KU;p5(ydpjxh@@uN*~=xpxtED0bc6cgi|&DQ9oY(1$S}z=Zil6!FII?f7{JJAnFoE ziWrzkUS0}lFw;cg@jtH~e*N^|FS1M&bpmRv=ZJ0H`TX~PQgTtuI-ZEh@IH^5kmPc4 zD?^*NYk@~Y0X9v#J$7WNmM?yAj})j=YggnU){(1?WbNsI044pcnTjbHh_2+C_75Ts zlv+kP?$=;k9~7rR@|BD4gm`;$ao)CDFRnsEjbM~tQYT8|C6$7eX;~c-y%`3VRD#bc+X5uMyI_qhVL6a@%VkGvS zntqXo;1U2afp~CCYi$qb$J>?4bUsY0IVgN=hMt4$>hR88gel-w9f3OdP5%)i5&5Ub zvNkIcx8{bU6~OMHmAc4+%91-j@m)gLGW_r~;v0%AgaSQ`83hfsgYNYB$vAO1zEy7n z;az z@fRr0@ZQ9G09dut>GYHj%JIXO^0$^>_s^zNtIt2R)bMc#TCraCJ*(1gI}Rc_O0B{x zEHoO|e|$OY&yLTkpJQ#x+4e7-c4+EfbXav=ega)3_oh$!$HOBidH(y4=c5bByC^Nbf#18} zQ08;}4dsE?d<3o?TtC7|2Zv-Y+|7^is1mYO*|Z)ZlLhV>zgi%=#ml~xZUO`^5n(^@ z2NY%0r`Z4bG`0LlURRXgFB=|-(8>-vkw~AS(J}kEv4O}EY(&Cl=RLlwE>vG-5N9j6 zwZN$ltaMW{xn}TaoWXK5qTj+J(TV??i8s`dF^43enoPM%vi8VC>c=~#EYp!XG!&;{ z91keG+{Le4d2jkRBvR?eHss=DAWUjQ#y9yi$~nAvMy9pohzc2IF6X6A#Pj@xd!BD> z00?^v0D3x;h40N}8}-v7u`vnaK68V=g{jSV?C!_w~lxW01XK{z^&HfIPwA(8Vm*Tpv#JX%D6PFO+Qw*~)UOl%^SK~IVdJZmEE^bAZVA)?Q; z${~D(H9lR=@$YC93*RN2E&RgBhwo}?(W#QIwD0Ig`zxcfI|p7MQ#u7Z`^Ki>Sk&vE<*(IQ`ypF0flSfIH;<@LQ~$* z%aaTia}}XEdI3{?j$H5u`^qKzA(Ij47-NBD-(9dj0#RfKulR5E2A1$v~wp2=I78fRj1`v4Om3Ws-ezyx@?5>P_fS_oO2&G#D6F7@-5Z8_$3A zuA5+05w4?dpmCWX0nPmZ^;Bw8D2hQ6TL`&^l88q&ks0omk%}pCIyWs)9Qm-;3mM#8 zE679y%rbwSjxGtAkF)K4nr7WNj_x7ce!OYA%as6dKax<@5Rwi8I;SByor{bS@Ze^4 zpFKKL0B>B#OA3 zf)SiL)3xG6VO6JgrQR4|y(ie=AG07r)!WQGfEZaU@+(k+Q~%Cy-b~f&CJcmQ!08{ot~tdL)B7Shj5dH?y{w)q9VBaJisHI?vNkdd zb?ziG!w2R!Vj#**L#KQ&QJP{{WT>S%smB#tI;L>Pj1vHFrf@d)F?F&Z5qTW34t2i8 zr>0BNo0lg>Lf`Wa%|sZ}EFf(J)Xtbvfw==vAmR7Rh*vtN9*Hfou%cAE$^G#z1qUP1 zc#ATjEg>xsm+A>zE?m1 zQVU3a1}QJK@jbYlq_t8rdD0k4hTa>dWr7wtg3`(Gc!&WVP90vLlut5v@=<;#pWU4$ zt>R^|S^`{BF)58z9pI#%7bTU{>B4Xen02M4g8Kq_2mV_>796R31q$v}0jb_F6#V>4 z1M2l@Bs0`M&9%^Qke10zQzt*GlN3o6Q*1CoTPd+g(-Fe6kN)zedRQV=on(50rq;f# z5&0lA?M*?F6g9{UUr@`Gq`8+bTJe^>bE`cX+|KcjHocv9?HZio3c-JC^z15%_bnqy zpC8;cL0qpfBto2-AT4TsGJkdziv%opF|f%u%oshVGh4Io;^dpVjzHHeM{~0jA~sxW zxH$bLj1bqAL46V82O;4??2wUpS|*i(y5+nS3gq)-8m{?suaTB%AF5aRcT>IO6I5ln z` z3HtvqGt5*1yzvS)Ruv`@O}VZ4IYbJS)hp|amsimM)visZ8br9{?b$J*f(uR+ze(Oy zyC6-fSjkpQDXJgCU8R_1ZhGbtAcUDf;`t_tJ)|C|KVLEr$_cZ8`aKdO@Ts!x^{=7& zCwXY8M$#bY_LSS%J;c|eC!W?|j*!We1}UF(aL=K-GPsj>pbUFr%0je;Erb5K+;)O> zG#FA)8j_Ge5DS>yaFq z@TS4hv_Bg>Wy1K`;$mI^WzkBWY_$q{ug)_Aae&3>Ns3pdmVt%7C)Yf^9u4(2;~O_| ql|R0#PCNO|7g>JA{~*oA7oc!>^7KbY@F74-TGtE2A|QL$jsFkg2J(Rb diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py new file mode 100644 index 0000000..14e872a --- /dev/null +++ b/youtube_dl/FileDownloader.py @@ -0,0 +1,689 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import httplib +import math +import os +import re +import socket +import subprocess +import sys +import time +import urllib2 + +if os.name == 'nt': + import ctypes + +from utils import * + + +class FileDownloader(object): + """File Downloader class. + + File downloader objects are the ones responsible of downloading the + actual video file and writing it to disk if the user has requested + it, among some other tasks. In most cases there should be one per + program. As, given a video URL, the downloader doesn't know how to + extract all the needed information, task that InfoExtractors do, it + has to pass the URL to one of them. + + For this, file downloader objects have a method that allows + InfoExtractors to be registered in a given order. When it is passed + a URL, the file downloader handles it to the first InfoExtractor it + finds that reports being able to handle it. The InfoExtractor extracts + all the information about the video or videos the URL refers to, and + asks the FileDownloader to process the video information, possibly + downloading the video. + + File downloaders accept a lot of parameters. In order not to saturate + the object constructor with arguments, it receives a dictionary of + options instead. These options are available through the params + attribute for the InfoExtractors to use. The FileDownloader also + registers itself as the downloader in charge for the InfoExtractors + that are added to it, so this is a "mutual registration". + + Available options: + + username: Username for authentication purposes. + password: Password for authentication purposes. + usenetrc: Use netrc for authentication instead. + quiet: Do not print messages to stdout. + forceurl: Force printing final URL. + forcetitle: Force printing title. + forcethumbnail: Force printing thumbnail URL. + forcedescription: Force printing description. + forcefilename: Force printing final filename. + simulate: Do not download the video files. + format: Video format code. + format_limit: Highest quality format to try. + outtmpl: Template for output names. + ignoreerrors: Do not stop on download errors. + ratelimit: Download speed limit, in bytes/sec. + nooverwrites: Prevent overwriting files. + retries: Number of times to retry for HTTP error 5xx + continuedl: Try to continue downloads if possible. + noprogress: Do not print the progress bar. + playliststart: Playlist item to start at. + playlistend: Playlist item to end at. + matchtitle: Download only matching titles. + rejecttitle: Reject downloads for matching titles. + logtostderr: Log messages to stderr instead of stdout. + consoletitle: Display progress in console window's titlebar. + nopart: Do not use temporary .part files. + updatetime: Use the Last-modified header to set output file timestamps. + writedescription: Write the video description to a .description file + writeinfojson: Write the video description to a .info.json file + writesubtitles: Write the video subtitles to a .srt file + subtitleslang: Language of the subtitles to download + """ + + params = None + _ies = [] + _pps = [] + _download_retcode = None + _num_downloads = None + _screen_file = None + + def __init__(self, params): + """Create a FileDownloader object with the given options.""" + self._ies = [] + self._pps = [] + self._download_retcode = 0 + self._num_downloads = 0 + self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] + self.params = params + + @staticmethod + def format_bytes(bytes): + if bytes is None: + return 'N/A' + if type(bytes) is str: + bytes = float(bytes) + if bytes == 0.0: + exponent = 0 + else: + exponent = long(math.log(bytes, 1024.0)) + suffix = 'bkMGTPEZY'[exponent] + converted = float(bytes) / float(1024 ** exponent) + return '%.2f%s' % (converted, suffix) + + @staticmethod + def calc_percent(byte_counter, data_len): + if data_len is None: + return '---.-%' + return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0)) + + @staticmethod + def calc_eta(start, now, total, current): + if total is None: + return '--:--' + dif = now - start + if current == 0 or dif < 0.001: # One millisecond + return '--:--' + rate = float(current) / dif + eta = long((float(total) - float(current)) / rate) + (eta_mins, eta_secs) = divmod(eta, 60) + if eta_mins > 99: + return '--:--' + return '%02d:%02d' % (eta_mins, eta_secs) + + @staticmethod + def calc_speed(start, now, bytes): + dif = now - start + if bytes == 0 or dif < 0.001: # One millisecond + return '%10s' % '---b/s' + return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif)) + + @staticmethod + def best_block_size(elapsed_time, bytes): + new_min = max(bytes / 2.0, 1.0) + new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB + if elapsed_time < 0.001: + return long(new_max) + rate = bytes / elapsed_time + if rate > new_max: + return long(new_max) + if rate < new_min: + return long(new_min) + return long(rate) + + @staticmethod + def parse_bytes(bytestr): + """Parse a string indicating a byte quantity into a long integer.""" + matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) + if matchobj is None: + return None + number = float(matchobj.group(1)) + multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) + return long(round(number * multiplier)) + + def add_info_extractor(self, ie): + """Add an InfoExtractor object to the end of the list.""" + self._ies.append(ie) + ie.set_downloader(self) + + def add_post_processor(self, pp): + """Add a PostProcessor object to the end of the chain.""" + self._pps.append(pp) + pp.set_downloader(self) + + def to_screen(self, message, skip_eol=False): + """Print message to stdout if not in quiet mode.""" + assert type(message) == type(u'') + if not self.params.get('quiet', False): + terminator = [u'\n', u''][skip_eol] + output = message + terminator + + if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr + output = output.encode(preferredencoding(), 'ignore') + self._screen_file.write(output) + self._screen_file.flush() + + def to_stderr(self, message): + """Print message to stderr.""" + print >>sys.stderr, message.encode(preferredencoding()) + + def to_cons_title(self, message): + """Set console/terminal window title to message.""" + if not self.params.get('consoletitle', False): + return + if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): + # c_wchar_p() might not be necessary if `message` is + # already of type unicode() + ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) + elif 'TERM' in os.environ: + sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding())) + + def fixed_template(self): + """Checks if the output template is fixed.""" + return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None) + + def trouble(self, message=None): + """Determine action to take when a download problem appears. + + Depending on if the downloader has been configured to ignore + download errors or not, this method may throw an exception or + not when errors are found, after printing the message. + """ + if message is not None: + self.to_stderr(message) + if not self.params.get('ignoreerrors', False): + raise DownloadError(message) + self._download_retcode = 1 + + def slow_down(self, start_time, byte_counter): + """Sleep if the download speed is over the rate limit.""" + rate_limit = self.params.get('ratelimit', None) + if rate_limit is None or byte_counter == 0: + return + now = time.time() + elapsed = now - start_time + if elapsed <= 0.0: + return + speed = float(byte_counter) / elapsed + if speed > rate_limit: + time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) + + def temp_name(self, filename): + """Returns a temporary filename for the given filename.""" + if self.params.get('nopart', False) or filename == u'-' or \ + (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): + return filename + return filename + u'.part' + + def undo_temp_name(self, filename): + if filename.endswith(u'.part'): + return filename[:-len(u'.part')] + return filename + + def try_rename(self, old_filename, new_filename): + try: + if old_filename == new_filename: + return + os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) + except (IOError, OSError), err: + self.trouble(u'ERROR: unable to rename file') + + def try_utime(self, filename, last_modified_hdr): + """Try to set the last-modified time of the given file.""" + if last_modified_hdr is None: + return + if not os.path.isfile(encodeFilename(filename)): + return + timestr = last_modified_hdr + if timestr is None: + return + filetime = timeconvert(timestr) + if filetime is None: + return filetime + try: + os.utime(filename, (time.time(), filetime)) + except: + pass + return filetime + + def report_writedescription(self, descfn): + """ Report that the description file is being written """ + self.to_screen(u'[info] Writing video description to: ' + descfn) + + def report_writesubtitles(self, srtfn): + """ Report that the subtitles file is being written """ + self.to_screen(u'[info] Writing video subtitles to: ' + srtfn) + + def report_writeinfojson(self, infofn): + """ Report that the metadata file has been written """ + self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) + + def report_destination(self, filename): + """Report destination filename.""" + self.to_screen(u'[download] Destination: ' + filename) + + def report_progress(self, percent_str, data_len_str, speed_str, eta_str): + """Report download progress.""" + if self.params.get('noprogress', False): + return + self.to_screen(u'\r[download] %s of %s at %s ETA %s' % + (percent_str, data_len_str, speed_str, eta_str), skip_eol=True) + self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' % + (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) + + def report_resuming_byte(self, resume_len): + """Report attempt to resume at given byte.""" + self.to_screen(u'[download] Resuming download at byte %s' % resume_len) + + def report_retry(self, count, retries): + """Report retry in case of HTTP error 5xx""" + self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) + + def report_file_already_downloaded(self, file_name): + """Report file has already been fully downloaded.""" + try: + self.to_screen(u'[download] %s has already been downloaded' % file_name) + except (UnicodeEncodeError), err: + self.to_screen(u'[download] The file has already been downloaded') + + def report_unable_to_resume(self): + """Report it was impossible to resume download.""" + self.to_screen(u'[download] Unable to resume') + + def report_finish(self): + """Report download finished.""" + if self.params.get('noprogress', False): + self.to_screen(u'[download] Download completed') + else: + self.to_screen(u'') + + def increment_downloads(self): + """Increment the ordinal that assigns a number to each file.""" + self._num_downloads += 1 + + def prepare_filename(self, info_dict): + """Generate the output filename.""" + try: + template_dict = dict(info_dict) + template_dict['epoch'] = unicode(long(time.time())) + template_dict['autonumber'] = unicode('%05d' % self._num_downloads) + filename = self.params['outtmpl'] % template_dict + return filename + except (ValueError, KeyError), err: + self.trouble(u'ERROR: invalid system charset or erroneous output template') + return None + + def _match_entry(self, info_dict): + """ Returns None iff the file should be downloaded """ + + title = info_dict['title'] + matchtitle = self.params.get('matchtitle', False) + if matchtitle and not re.search(matchtitle, title, re.IGNORECASE): + return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' + rejecttitle = self.params.get('rejecttitle', False) + if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE): + return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' + return None + + def process_info(self, info_dict): + """Process a single dictionary returned by an InfoExtractor.""" + + info_dict['stitle'] = sanitize_filename(info_dict['title']) + + reason = self._match_entry(info_dict) + if reason is not None: + self.to_screen(u'[download] ' + reason) + return + + max_downloads = self.params.get('max_downloads') + if max_downloads is not None: + if self._num_downloads > int(max_downloads): + raise MaxDownloadsReached() + + filename = self.prepare_filename(info_dict) + + # Forced printings + if self.params.get('forcetitle', False): + print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace') + if self.params.get('forceurl', False): + print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace') + if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict: + print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace') + if self.params.get('forcedescription', False) and 'description' in info_dict: + print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace') + if self.params.get('forcefilename', False) and filename is not None: + print filename.encode(preferredencoding(), 'xmlcharrefreplace') + if self.params.get('forceformat', False): + print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace') + + # Do nothing else if in simulate mode + if self.params.get('simulate', False): + return + + if filename is None: + return + + try: + dn = os.path.dirname(encodeFilename(filename)) + if dn != '' and not os.path.exists(dn): # dn is already encoded + os.makedirs(dn) + except (OSError, IOError), err: + self.trouble(u'ERROR: unable to create directory ' + unicode(err)) + return + + if self.params.get('writedescription', False): + try: + descfn = filename + u'.description' + self.report_writedescription(descfn) + descfile = open(encodeFilename(descfn), 'wb') + try: + descfile.write(info_dict['description'].encode('utf-8')) + finally: + descfile.close() + except (OSError, IOError): + self.trouble(u'ERROR: Cannot write description file ' + descfn) + return + + if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']: + # subtitles download errors are already managed as troubles in relevant IE + # that way it will silently go on when used with unsupporting IE + try: + srtfn = filename.rsplit('.', 1)[0] + u'.srt' + self.report_writesubtitles(srtfn) + srtfile = open(encodeFilename(srtfn), 'wb') + try: + srtfile.write(info_dict['subtitles'].encode('utf-8')) + finally: + srtfile.close() + except (OSError, IOError): + self.trouble(u'ERROR: Cannot write subtitles file ' + descfn) + return + + if self.params.get('writeinfojson', False): + infofn = filename + u'.info.json' + self.report_writeinfojson(infofn) + try: + json.dump + except (NameError,AttributeError): + self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.') + return + try: + infof = open(encodeFilename(infofn), 'wb') + try: + json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',)) + json.dump(json_info_dict, infof) + finally: + infof.close() + except (OSError, IOError): + self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn) + return + + if not self.params.get('skip_download', False): + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)): + success = True + else: + try: + success = self._do_download(filename, info_dict) + except (OSError, IOError), err: + raise UnavailableVideoError + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self.trouble(u'ERROR: unable to download video data: %s' % str(err)) + return + except (ContentTooShortError, ), err: + self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) + return + + if success: + try: + self.post_process(filename, info_dict) + except (PostProcessingError), err: + self.trouble(u'ERROR: postprocessing: %s' % str(err)) + return + + def download(self, url_list): + """Download a given list of URLs.""" + if len(url_list) > 1 and self.fixed_template(): + raise SameFileError(self.params['outtmpl']) + + for url in url_list: + suitable_found = False + for ie in self._ies: + # Go to next InfoExtractor if not suitable + if not ie.suitable(url): + continue + + # Suitable InfoExtractor found + suitable_found = True + + # Extract information from URL and process it + videos = ie.extract(url) + for video in videos or []: + try: + self.increment_downloads() + self.process_info(video) + except UnavailableVideoError: + self.trouble(u'\nERROR: unable to download video') + + # Suitable InfoExtractor had been found; go to next URL + break + + if not suitable_found: + self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) + + return self._download_retcode + + def post_process(self, filename, ie_info): + """Run the postprocessing chain on the given file.""" + info = dict(ie_info) + info['filepath'] = filename + for pp in self._pps: + info = pp.run(info) + if info is None: + break + + def _download_with_rtmpdump(self, filename, url, player_url): + self.report_destination(filename) + tmpfilename = self.temp_name(filename) + + # Check for rtmpdump first + try: + subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT) + except (OSError, IOError): + self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run') + return False + + # Download using rtmpdump. rtmpdump returns exit code 2 when + # the connection was interrumpted and resuming appears to be + # possible. This is part of rtmpdump's normal usage, AFAIK. + basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename] + args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)] + if self.params.get('verbose', False): + try: + import pipes + shell_quote = lambda args: ' '.join(map(pipes.quote, args)) + except ImportError: + shell_quote = repr + self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args)) + retval = subprocess.call(args) + while retval == 2 or retval == 1: + prevsize = os.path.getsize(encodeFilename(tmpfilename)) + self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) + time.sleep(5.0) # This seems to be needed + retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) + cursize = os.path.getsize(encodeFilename(tmpfilename)) + if prevsize == cursize and retval == 1: + break + # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those + if prevsize == cursize and retval == 2 and cursize > 1024: + self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.') + retval = 0 + break + if retval == 0: + self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename))) + self.try_rename(tmpfilename, filename) + return True + else: + self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval) + return False + + def _do_download(self, filename, info_dict): + url = info_dict['url'] + player_url = info_dict.get('player_url', None) + + # Check file already present + if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False): + self.report_file_already_downloaded(filename) + return True + + # Attempt to download using rtmpdump + if url.startswith('rtmp'): + return self._download_with_rtmpdump(filename, url, player_url) + + tmpfilename = self.temp_name(filename) + stream = None + + # Do not include the Accept-Encoding header + headers = {'Youtubedl-no-compression': 'True'} + basic_request = urllib2.Request(url, None, headers) + request = urllib2.Request(url, None, headers) + + # Establish possible resume length + if os.path.isfile(encodeFilename(tmpfilename)): + resume_len = os.path.getsize(encodeFilename(tmpfilename)) + else: + resume_len = 0 + + open_mode = 'wb' + if resume_len != 0: + if self.params.get('continuedl', False): + self.report_resuming_byte(resume_len) + request.add_header('Range','bytes=%d-' % resume_len) + open_mode = 'ab' + else: + resume_len = 0 + + count = 0 + retries = self.params.get('retries', 0) + while count <= retries: + # Establish connection + try: + if count == 0 and 'urlhandle' in info_dict: + data = info_dict['urlhandle'] + data = urllib2.urlopen(request) + break + except (urllib2.HTTPError, ), err: + if (err.code < 500 or err.code >= 600) and err.code != 416: + # Unexpected HTTP error + raise + elif err.code == 416: + # Unable to resume (requested range not satisfiable) + try: + # Open the connection again without the range header + data = urllib2.urlopen(basic_request) + content_length = data.info()['Content-Length'] + except (urllib2.HTTPError, ), err: + if err.code < 500 or err.code >= 600: + raise + else: + # Examine the reported length + if (content_length is not None and + (resume_len - 100 < long(content_length) < resume_len + 100)): + # The file had already been fully downloaded. + # Explanation to the above condition: in issue #175 it was revealed that + # YouTube sometimes adds or removes a few bytes from the end of the file, + # changing the file size slightly and causing problems for some users. So + # I decided to implement a suggested change and consider the file + # completely downloaded if the file size differs less than 100 bytes from + # the one in the hard drive. + self.report_file_already_downloaded(filename) + self.try_rename(tmpfilename, filename) + return True + else: + # The length does not match, we start the download over + self.report_unable_to_resume() + open_mode = 'wb' + break + # Retry + count += 1 + if count <= retries: + self.report_retry(count, retries) + + if count > retries: + self.trouble(u'ERROR: giving up after %s retries' % retries) + return False + + data_len = data.info().get('Content-length', None) + if data_len is not None: + data_len = long(data_len) + resume_len + data_len_str = self.format_bytes(data_len) + byte_counter = 0 + resume_len + block_size = 1024 + start = time.time() + while True: + # Download and write + before = time.time() + data_block = data.read(block_size) + after = time.time() + if len(data_block) == 0: + break + byte_counter += len(data_block) + + # Open file just in time + if stream is None: + try: + (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) + assert stream is not None + filename = self.undo_temp_name(tmpfilename) + self.report_destination(filename) + except (OSError, IOError), err: + self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) + return False + try: + stream.write(data_block) + except (IOError, OSError), err: + self.trouble(u'\nERROR: unable to write data: %s' % str(err)) + return False + block_size = self.best_block_size(after - before, len(data_block)) + + # Progress message + speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len) + if data_len is None: + self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA') + else: + percent_str = self.calc_percent(byte_counter, data_len) + eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len) + self.report_progress(percent_str, data_len_str, speed_str, eta_str) + + # Apply rate limit + self.slow_down(start, byte_counter - resume_len) + + if stream is None: + self.trouble(u'\nERROR: Did not get any data blocks') + return False + stream.close() + self.report_finish() + if data_len is not None and byte_counter != data_len: + raise ContentTooShortError(byte_counter, long(data_len)) + self.try_rename(tmpfilename, filename) + + # Update file modification time + if self.params.get('updatetime', True): + info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None)) + + return True diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py new file mode 100644 index 0000000..ddb4aa1 --- /dev/null +++ b/youtube_dl/InfoExtractors.py @@ -0,0 +1,2957 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import datetime +import HTMLParser +import httplib +import netrc +import os +import re +import socket +import time +import urllib +import urllib2 +import email.utils +import xml.etree.ElementTree +from urlparse import parse_qs + +try: + import cStringIO as StringIO +except ImportError: + import StringIO + +from utils import * + + +class InfoExtractor(object): + """Information Extractor class. + + Information extractors are the classes that, given a URL, extract + information from the video (or videos) the URL refers to. This + information includes the real video URL, the video title and simplified + title, author and others. The information is stored in a dictionary + which is then passed to the FileDownloader. The FileDownloader + processes this information possibly downloading the video to the file + system, among other possible outcomes. The dictionaries must include + the following fields: + + id: Video identifier. + url: Final video URL. + uploader: Nickname of the video uploader. + title: Literal title. + ext: Video filename extension. + format: Video format. + player_url: SWF Player URL (may be None). + + The following fields are optional. Their primary purpose is to allow + youtube-dl to serve as the backend for a video search function, such + as the one in youtube2mp3. They are only used when their respective + forced printing functions are called: + + thumbnail: Full URL to a video thumbnail image. + description: One-line video description. + + Subclasses of this one should re-define the _real_initialize() and + _real_extract() methods and define a _VALID_URL regexp. + Probably, they should also be added to the list of extractors. + """ + + _ready = False + _downloader = None + + def __init__(self, downloader=None): + """Constructor. Receives an optional downloader.""" + self._ready = False + self.set_downloader(downloader) + + def suitable(self, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(self._VALID_URL, url) is not None + + def initialize(self): + """Initializes an instance (authentication, etc).""" + if not self._ready: + self._real_initialize() + self._ready = True + + def extract(self, url): + """Extracts URL information and returns it in list of dicts.""" + self.initialize() + return self._real_extract(url) + + def set_downloader(self, downloader): + """Sets the downloader for this IE.""" + self._downloader = downloader + + def _real_initialize(self): + """Real initialization process. Redefine in subclasses.""" + pass + + def _real_extract(self, url): + """Real extraction process. Redefine in subclasses.""" + pass + + +class YoutubeIE(InfoExtractor): + """Information extractor for youtube.com.""" + + _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?!view_play_list|my_playlists|artist|playlist)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$' + _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' + _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en' + _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' + _NEXT_URL_RE = r'[\?&]next_url=([^&]+)' + _NETRC_MACHINE = 'youtube' + # Listed in order of quality + _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13'] + _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13'] + _video_extensions = { + '13': '3gp', + '17': 'mp4', + '18': 'mp4', + '22': 'mp4', + '37': 'mp4', + '38': 'video', # You actually don't know if this will be MOV, AVI or whatever + '43': 'webm', + '44': 'webm', + '45': 'webm', + '46': 'webm', + } + _video_dimensions = { + '5': '240x400', + '6': '???', + '13': '???', + '17': '144x176', + '18': '360x640', + '22': '720x1280', + '34': '360x640', + '35': '480x854', + '37': '1080x1920', + '38': '3072x4096', + '43': '360x640', + '44': '480x854', + '45': '720x1280', + '46': '1080x1920', + } + IE_NAME = u'youtube' + + def report_lang(self): + """Report attempt to set language.""" + self._downloader.to_screen(u'[youtube] Setting language') + + def report_login(self): + """Report attempt to log in.""" + self._downloader.to_screen(u'[youtube] Logging in') + + def report_age_confirmation(self): + """Report attempt to confirm age.""" + self._downloader.to_screen(u'[youtube] Confirming age') + + def report_video_webpage_download(self, video_id): + """Report attempt to download video webpage.""" + self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id) + + def report_video_info_webpage_download(self, video_id): + """Report attempt to download video info webpage.""" + self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id) + + def report_video_subtitles_download(self, video_id): + """Report attempt to download video info webpage.""" + self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id) + + def report_information_extraction(self, video_id): + """Report attempt to extract video information.""" + self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id) + + def report_unavailable_format(self, video_id, format): + """Report extracted video URL.""" + self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format)) + + def report_rtmp_download(self): + """Indicate the download will use the RTMP protocol.""" + self._downloader.to_screen(u'[youtube] RTMP download detected') + + def _closed_captions_xml_to_srt(self, xml_string): + srt = '' + texts = re.findall(r'([^<]+)', xml_string, re.MULTILINE) + # TODO parse xml instead of regex + for n, (start, dur_tag, dur, caption) in enumerate(texts): + if not dur: dur = '4' + start = float(start) + end = start + float(dur) + start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000) + end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000) + caption = unescapeHTML(caption) + caption = unescapeHTML(caption) # double cycle, intentional + srt += str(n+1) + '\n' + srt += start + ' --> ' + end + '\n' + srt += caption + '\n\n' + return srt + + def _print_formats(self, formats): + print 'Available formats:' + for x in formats: + print '%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')) + + def _real_initialize(self): + if self._downloader is None: + return + + username = None + password = None + downloader_params = self._downloader.params + + # Attempt to use provided username and password or .netrc data + if downloader_params.get('username', None) is not None: + username = downloader_params['username'] + password = downloader_params['password'] + elif downloader_params.get('usenetrc', False): + try: + info = netrc.netrc().authenticators(self._NETRC_MACHINE) + if info is not None: + username = info[0] + password = info[2] + else: + raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) + except (IOError, netrc.NetrcParseError), err: + self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err)) + return + + # Set language + request = urllib2.Request(self._LANG_URL) + try: + self.report_lang() + urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err)) + return + + # No authentication to be performed + if username is None: + return + + # Log in + login_form = { + 'current_form': 'loginForm', + 'next': '/', + 'action_login': 'Log In', + 'username': username, + 'password': password, + } + request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form)) + try: + self.report_login() + login_results = urllib2.urlopen(request).read() + if re.search(r'(?i)]* name="loginForm"', login_results) is not None: + self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password') + return + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err)) + return + + # Confirm age + age_form = { + 'next_url': '/', + 'action_confirm': 'Confirm', + } + request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form)) + try: + self.report_age_confirmation() + age_results = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err)) + return + + def _real_extract(self, url): + # Extract original video URL from URL with redirection, like age verification, using next_url parameter + mobj = re.search(self._NEXT_URL_RE, url) + if mobj: + url = 'http://www.youtube.com/' + urllib.unquote(mobj.group(1)).lstrip('/') + + # Extract video id from URL + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + video_id = mobj.group(2) + + # Get video webpage + self.report_video_webpage_download(video_id) + request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id) + try: + video_webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) + return + + # Attempt to extract SWF player URL + mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage) + if mobj is not None: + player_url = re.sub(r'\\(.)', r'\1', mobj.group(1)) + else: + player_url = None + + # Get video info + self.report_video_info_webpage_download(video_id) + for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']: + video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en' + % (video_id, el_type)) + request = urllib2.Request(video_info_url) + try: + video_info_webpage = urllib2.urlopen(request).read() + video_info = parse_qs(video_info_webpage) + if 'token' in video_info: + break + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err)) + return + if 'token' not in video_info: + if 'reason' in video_info: + self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8')) + else: + self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason') + return + + # Check for "rental" videos + if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info: + self._downloader.trouble(u'ERROR: "rental" videos not supported') + return + + # Start extracting information + self.report_information_extraction(video_id) + + # uploader + if 'author' not in video_info: + self._downloader.trouble(u'ERROR: unable to extract uploader nickname') + return + video_uploader = urllib.unquote_plus(video_info['author'][0]) + + # title + if 'title' not in video_info: + self._downloader.trouble(u'ERROR: unable to extract video title') + return + video_title = urllib.unquote_plus(video_info['title'][0]) + video_title = video_title.decode('utf-8') + + # thumbnail image + if 'thumbnail_url' not in video_info: + self._downloader.trouble(u'WARNING: unable to extract video thumbnail') + video_thumbnail = '' + else: # don't panic if we can't find it + video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0]) + + # upload date + upload_date = u'NA' + mobj = re.search(r'id="eow-date.*?>(.*?)', video_webpage, re.DOTALL) + if mobj is not None: + upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split()) + format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y'] + for expression in format_expressions: + try: + upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d') + except: + pass + + # description + video_description = get_element_by_id("eow-description", video_webpage.decode('utf8')) + if video_description: video_description = clean_html(video_description) + else: video_description = '' + + # closed captions + video_subtitles = None + if self._downloader.params.get('writesubtitles', False): + try: + self.report_video_subtitles_download(video_id) + request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id) + try: + srt_list = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + raise Trouble(u'WARNING: unable to download video subtitles: %s' % str(err)) + srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list) + srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list) + if not srt_lang_list: + raise Trouble(u'WARNING: video has no closed captions') + if self._downloader.params.get('subtitleslang', False): + srt_lang = self._downloader.params.get('subtitleslang') + elif 'en' in srt_lang_list: + srt_lang = 'en' + else: + srt_lang = srt_lang_list.keys()[0] + if not srt_lang in srt_lang_list: + raise Trouble(u'WARNING: no closed captions found in the specified language') + request = urllib2.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id)) + try: + srt_xml = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + raise Trouble(u'WARNING: unable to download video subtitles: %s' % str(err)) + if not srt_xml: + raise Trouble(u'WARNING: unable to download video subtitles') + video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8')) + except Trouble as trouble: + self._downloader.trouble(trouble[0]) + + # token + video_token = urllib.unquote_plus(video_info['token'][0]) + + # Decide which formats to download + req_format = self._downloader.params.get('format', None) + + if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'): + self.report_rtmp_download() + video_url_list = [(None, video_info['conn'][0])] + elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1: + url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',') + url_data = [parse_qs(uds) for uds in url_data_strs] + url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data) + url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data) + + format_limit = self._downloader.params.get('format_limit', None) + available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats + if format_limit is not None and format_limit in available_formats: + format_list = available_formats[available_formats.index(format_limit):] + else: + format_list = available_formats + existing_formats = [x for x in format_list if x in url_map] + if len(existing_formats) == 0: + self._downloader.trouble(u'ERROR: no known formats available for video') + return + if self._downloader.params.get('listformats', None): + self._print_formats(existing_formats) + return + if req_format is None or req_format == 'best': + video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality + elif req_format == 'worst': + video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality + elif req_format in ('-1', 'all'): + video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats + else: + # Specific formats. We pick the first in a slash-delimeted sequence. + # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'. + req_formats = req_format.split('/') + video_url_list = None + for rf in req_formats: + if rf in url_map: + video_url_list = [(rf, url_map[rf])] + break + if video_url_list is None: + self._downloader.trouble(u'ERROR: requested format not available') + return + else: + self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info') + return + + results = [] + for format_param, video_real_url in video_url_list: + # Extension + video_extension = self._video_extensions.get(format_param, 'flv') + + results.append({ + 'id': video_id.decode('utf-8'), + 'url': video_real_url.decode('utf-8'), + 'uploader': video_uploader.decode('utf-8'), + 'upload_date': upload_date, + 'title': video_title, + 'ext': video_extension.decode('utf-8'), + 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), + 'thumbnail': video_thumbnail.decode('utf-8'), + 'description': video_description, + 'player_url': player_url, + 'subtitles': video_subtitles + }) + return results + + +class MetacafeIE(InfoExtractor): + """Information Extractor for metacafe.com.""" + + _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*' + _DISCLAIMER = 'http://www.metacafe.com/family_filter/' + _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user' + IE_NAME = u'metacafe' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_disclaimer(self): + """Report disclaimer retrieval.""" + self._downloader.to_screen(u'[metacafe] Retrieving disclaimer') + + def report_age_confirmation(self): + """Report attempt to confirm age.""" + self._downloader.to_screen(u'[metacafe] Confirming age') + + def report_download_webpage(self, video_id): + """Report webpage download.""" + self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id) + + def _real_initialize(self): + # Retrieve disclaimer + request = urllib2.Request(self._DISCLAIMER) + try: + self.report_disclaimer() + disclaimer = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err)) + return + + # Confirm age + disclaimer_form = { + 'filters': '0', + 'submit': "Continue - I'm over 18", + } + request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form)) + try: + self.report_age_confirmation() + disclaimer = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err)) + return + + def _real_extract(self, url): + # Extract id and simplified title from URL + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group(1) + + # Check if video comes from YouTube + mobj2 = re.match(r'^yt-(.*)$', video_id) + if mobj2 is not None: + self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)]) + return + + # Retrieve video webpage to extract further information + request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id) + try: + self.report_download_webpage(video_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err)) + return + + # Extract URL, uploader and title from webpage + self.report_extraction(video_id) + mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage) + if mobj is not None: + mediaURL = urllib.unquote(mobj.group(1)) + video_extension = mediaURL[-3:] + + # Extract gdaKey if available + mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage) + if mobj is None: + video_url = mediaURL + else: + gdaKey = mobj.group(1) + video_url = '%s?__gda__=%s' % (mediaURL, gdaKey) + else: + mobj = re.search(r' name="flashvars" value="(.*?)"', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract media URL') + return + vardict = parse_qs(mobj.group(1)) + if 'mediaData' not in vardict: + self._downloader.trouble(u'ERROR: unable to extract media URL') + return + mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0]) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract media URL') + return + mediaURL = mobj.group(1).replace('\\/', '/') + video_extension = mediaURL[-3:] + video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2)) + + mobj = re.search(r'(?im)(.*) - Video', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + video_title = mobj.group(1).decode('utf-8') + + mobj = re.search(r'(?ms)By:\s*(.+?)<', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract uploader nickname') + return + video_uploader = mobj.group(1) + + return [{ + 'id': video_id.decode('utf-8'), + 'url': video_url.decode('utf-8'), + 'uploader': video_uploader.decode('utf-8'), + 'upload_date': u'NA', + 'title': video_title, + 'ext': video_extension.decode('utf-8'), + 'format': u'NA', + 'player_url': None, + }] + + +class DailymotionIE(InfoExtractor): + """Information Extractor for Dailymotion""" + + _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)' + IE_NAME = u'dailymotion' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_webpage(self, video_id): + """Report webpage download.""" + self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id) + + def _real_extract(self, url): + # Extract id and simplified title from URL + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group(1) + + video_extension = 'flv' + + # Retrieve video webpage to extract further information + request = urllib2.Request(url) + request.add_header('Cookie', 'family_filter=off') + try: + self.report_download_webpage(video_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err)) + return + + # Extract URL, uploader and title from webpage + self.report_extraction(video_id) + mobj = re.search(r'(?i)addVariable\(\"sequence\"\s*,\s*\"([^\"]+?)\"\)', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract media URL') + return + sequence = urllib.unquote(mobj.group(1)) + mobj = re.search(r',\"sdURL\"\:\"([^\"]+?)\",', sequence) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract media URL') + return + mediaURL = urllib.unquote(mobj.group(1)).replace('\\', '') + + # if needed add http://www.dailymotion.com/ if relative URL + + video_url = mediaURL + + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + video_title = unescapeHTML(mobj.group('title').decode('utf-8')) + + mobj = re.search(r'(?im)[^<]+?]+?>([^<]+?)', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract uploader nickname') + return + video_uploader = mobj.group(1) + + return [{ + 'id': video_id.decode('utf-8'), + 'url': video_url.decode('utf-8'), + 'uploader': video_uploader.decode('utf-8'), + 'upload_date': u'NA', + 'title': video_title, + 'ext': video_extension.decode('utf-8'), + 'format': u'NA', + 'player_url': None, + }] + + +class GoogleIE(InfoExtractor): + """Information extractor for video.google.com.""" + + _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*' + IE_NAME = u'video.google' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_webpage(self, video_id): + """Report webpage download.""" + self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id) + + def _real_extract(self, url): + # Extract id from URL + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + return + + video_id = mobj.group(1) + + video_extension = 'mp4' + + # Retrieve video webpage to extract further information + request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id) + try: + self.report_download_webpage(video_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) + return + + # Extract URL, uploader, and title from webpage + self.report_extraction(video_id) + mobj = re.search(r"download_url:'([^']+)'", webpage) + if mobj is None: + video_extension = 'flv' + mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract media URL') + return + mediaURL = urllib.unquote(mobj.group(1)) + mediaURL = mediaURL.replace('\\x3d', '\x3d') + mediaURL = mediaURL.replace('\\x26', '\x26') + + video_url = mediaURL + + mobj = re.search(r'(.*)', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + video_title = mobj.group(1).decode('utf-8') + + # Extract video description + mobj = re.search(r'([^<]*)', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video description') + return + video_description = mobj.group(1).decode('utf-8') + if not video_description: + video_description = 'No description available.' + + # Extract video thumbnail + if self._downloader.params.get('forcethumbnail', False): + request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id))) + try: + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) + return + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video thumbnail') + return + video_thumbnail = mobj.group(1) + else: # we need something to pass to process_info + video_thumbnail = '' + + return [{ + 'id': video_id.decode('utf-8'), + 'url': video_url.decode('utf-8'), + 'uploader': u'NA', + 'upload_date': u'NA', + 'title': video_title, + 'ext': video_extension.decode('utf-8'), + 'format': u'NA', + 'player_url': None, + }] + + +class PhotobucketIE(InfoExtractor): + """Information extractor for photobucket.com.""" + + _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)' + IE_NAME = u'photobucket' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_webpage(self, video_id): + """Report webpage download.""" + self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id) + + def _real_extract(self, url): + # Extract id from URL + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + return + + video_id = mobj.group(1) + + video_extension = 'flv' + + # Retrieve video webpage to extract further information + request = urllib2.Request(url) + try: + self.report_download_webpage(video_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) + return + + # Extract URL, uploader, and title from webpage + self.report_extraction(video_id) + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract media URL') + return + mediaURL = urllib.unquote(mobj.group(1)) + + video_url = mediaURL + + mobj = re.search(r'(.*) video by (.*) - Photobucket', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + video_title = mobj.group(1).decode('utf-8') + + video_uploader = mobj.group(2).decode('utf-8') + + return [{ + 'id': video_id.decode('utf-8'), + 'url': video_url.decode('utf-8'), + 'uploader': video_uploader, + 'upload_date': u'NA', + 'title': video_title, + 'ext': video_extension.decode('utf-8'), + 'format': u'NA', + 'player_url': None, + }] + + +class YahooIE(InfoExtractor): + """Information extractor for video.yahoo.com.""" + + # _VALID_URL matches all Yahoo! Video URLs + # _VPAGE_URL matches only the extractable '/watch/' URLs + _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?' + _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?' + IE_NAME = u'video.yahoo' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_webpage(self, video_id): + """Report webpage download.""" + self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id) + + def _real_extract(self, url, new_video=True): + # Extract ID from URL + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + return + + video_id = mobj.group(2) + video_extension = 'flv' + + # Rewrite valid but non-extractable URLs as + # extractable English language /watch/ URLs + if re.match(self._VPAGE_URL, url) is None: + request = urllib2.Request(url) + try: + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) + return + + mobj = re.search(r'\("id", "([0-9]+)"\);', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: Unable to extract id field') + return + yahoo_id = mobj.group(1) + + mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: Unable to extract vid field') + return + yahoo_vid = mobj.group(1) + + url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id) + return self._real_extract(url, new_video=False) + + # Retrieve video webpage to extract further information + request = urllib2.Request(url) + try: + self.report_download_webpage(video_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) + return + + # Extract uploader and title from webpage + self.report_extraction(video_id) + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video title') + return + video_title = mobj.group(1).decode('utf-8') + + mobj = re.search(r'

(.*)

', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video uploader') + return + video_uploader = mobj.group(1).decode('utf-8') + + # Extract video thumbnail + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video thumbnail') + return + video_thumbnail = mobj.group(1).decode('utf-8') + + # Extract video description + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video description') + return + video_description = mobj.group(1).decode('utf-8') + if not video_description: + video_description = 'No description available.' + + # Extract video height and width + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video height') + return + yv_video_height = mobj.group(1) + + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video width') + return + yv_video_width = mobj.group(1) + + # Retrieve video playlist to extract media URL + # I'm not completely sure what all these options are, but we + # seem to need most of them, otherwise the server sends a 401. + yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents + yv_bitrate = '700' # according to Wikipedia this is hard-coded + request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id + + '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height + + '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797') + try: + self.report_download_webpage(video_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) + return + + # Extract media URL from playlist XML + mobj = re.search(r'[^:]*: (.*?)( \([^\(]*\))?', webpage) + if mobj is not None: + video_upload_date = mobj.group(1) + + # Vimeo specific: extract request signature and timestamp + sig = config['request']['signature'] + timestamp = config['request']['timestamp'] + + # Vimeo specific: extract video codec and quality information + # TODO bind to format param + codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')] + for codec in codecs: + if codec[0] in config["video"]["files"]: + video_codec = codec[0] + video_extension = codec[1] + if 'hd' in config["video"]["files"][codec[0]]: quality = 'hd' + else: quality = 'sd' + break + else: + self._downloader.trouble(u'ERROR: no known codec found') + return + + video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \ + %(video_id, sig, timestamp, quality, video_codec.upper()) + + return [{ + 'id': video_id, + 'url': video_url, + 'uploader': video_uploader, + 'upload_date': video_upload_date, + 'title': video_title, + 'ext': video_extension, + 'thumbnail': video_thumbnail, + 'description': video_description, + 'player_url': None, + }] + + +class GenericIE(InfoExtractor): + """Generic last-resort information extractor.""" + + _VALID_URL = r'.*' + IE_NAME = u'generic' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_webpage(self, video_id): + """Report webpage download.""" + self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.') + self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id) + + def report_following_redirect(self, new_url): + """Report information extraction.""" + self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) + + def _test_redirect(self, url): + """Check if it is a redirect, like url shorteners, in case restart chain.""" + class HeadRequest(urllib2.Request): + def get_method(self): + return "HEAD" + + class HEADRedirectHandler(urllib2.HTTPRedirectHandler): + """ + Subclass the HTTPRedirectHandler to make it use our + HeadRequest also on the redirected URL + """ + def redirect_request(self, req, fp, code, msg, headers, newurl): + if code in (301, 302, 303, 307): + newurl = newurl.replace(' ', '%20') + newheaders = dict((k,v) for k,v in req.headers.items() + if k.lower() not in ("content-length", "content-type")) + return HeadRequest(newurl, + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True) + else: + raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) + + class HTTPMethodFallback(urllib2.BaseHandler): + """ + Fallback to GET if HEAD is not allowed (405 HTTP error) + """ + def http_error_405(self, req, fp, code, msg, headers): + fp.read() + fp.close() + + newheaders = dict((k,v) for k,v in req.headers.items() + if k.lower() not in ("content-length", "content-type")) + return self.parent.open(urllib2.Request(req.get_full_url(), + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True)) + + # Build our opener + opener = urllib2.OpenerDirector() + for handler in [urllib2.HTTPHandler, urllib2.HTTPDefaultErrorHandler, + HTTPMethodFallback, HEADRedirectHandler, + urllib2.HTTPErrorProcessor, urllib2.HTTPSHandler]: + opener.add_handler(handler()) + + response = opener.open(HeadRequest(url)) + new_url = response.geturl() + + if url == new_url: return False + + self.report_following_redirect(new_url) + self._downloader.download([new_url]) + return True + + def _real_extract(self, url): + if self._test_redirect(url): return + + video_id = url.split('/')[-1] + request = urllib2.Request(url) + try: + self.report_download_webpage(video_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) + return + except ValueError, err: + # since this is the last-resort InfoExtractor, if + # this error is thrown, it'll be thrown here + self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + return + + self.report_extraction(video_id) + # Start with something easy: JW Player in SWFObject + mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) + if mobj is None: + # Broaden the search a little bit + mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + return + + # It's possible that one of the regexes + # matched, but returned an empty group: + if mobj.group(1) is None: + self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) + return + + video_url = urllib.unquote(mobj.group(1)) + video_id = os.path.basename(video_url) + + # here's a fun little line of code for you: + video_extension = os.path.splitext(video_id)[1][1:] + video_id = os.path.splitext(video_id)[0] + + # it's tempting to parse this further, but you would + # have to take into account all the variations like + # Video Title - Site Name + # Site Name | Video Title + # Video Title - Tagline | Site Name + # and so on and so forth; it's just not practical + mobj = re.search(r'(.*)', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + video_title = mobj.group(1).decode('utf-8') + + # video uploader is domain name + mobj = re.match(r'(?:https?://)?([^/]*)/.*', url) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + video_uploader = mobj.group(1).decode('utf-8') + + return [{ + 'id': video_id.decode('utf-8'), + 'url': video_url.decode('utf-8'), + 'uploader': video_uploader, + 'upload_date': u'NA', + 'title': video_title, + 'ext': video_extension.decode('utf-8'), + 'format': u'NA', + 'player_url': None, + }] + + +class YoutubeSearchIE(InfoExtractor): + """Information Extractor for YouTube search queries.""" + _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+' + _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc' + _max_youtube_results = 1000 + IE_NAME = u'youtube:search' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_page(self, query, pagenum): + """Report attempt to download search page with given number.""" + query = query.decode(preferredencoding()) + self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum)) + + def _real_extract(self, query): + mobj = re.match(self._VALID_URL, query) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) + return + + prefix, query = query.split(':') + prefix = prefix[8:] + query = query.encode('utf-8') + if prefix == '': + self._download_n_results(query, 1) + return + elif prefix == 'all': + self._download_n_results(query, self._max_youtube_results) + return + else: + try: + n = long(prefix) + if n <= 0: + self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) + return + elif n > self._max_youtube_results: + self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n)) + n = self._max_youtube_results + self._download_n_results(query, n) + return + except ValueError: # parsing prefix as integer fails + self._download_n_results(query, 1) + return + + def _download_n_results(self, query, n): + """Downloads a specified number of results for a query""" + + video_ids = [] + pagenum = 0 + limit = n + + while (50 * pagenum) < limit: + self.report_download_page(query, pagenum+1) + result_url = self._API_URL % (urllib.quote_plus(query), (50*pagenum)+1) + request = urllib2.Request(result_url) + try: + data = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download API page: %s' % str(err)) + return + api_response = json.loads(data)['data'] + + new_ids = list(video['id'] for video in api_response['items']) + video_ids += new_ids + + limit = min(n, api_response['totalItems']) + pagenum += 1 + + if len(video_ids) > n: + video_ids = video_ids[:n] + for id in video_ids: + self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) + return + + +class GoogleSearchIE(InfoExtractor): + """Information Extractor for Google Video search queries.""" + _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+' + _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en' + _VIDEO_INDICATOR = r' self._max_google_results: + self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n)) + n = self._max_google_results + self._download_n_results(query, n) + return + except ValueError: # parsing prefix as integer fails + self._download_n_results(query, 1) + return + + def _download_n_results(self, query, n): + """Downloads a specified number of results for a query""" + + video_ids = [] + pagenum = 0 + + while True: + self.report_download_page(query, pagenum) + result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum*10) + request = urllib2.Request(result_url) + try: + page = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) + return + + # Extract video identifiers + for mobj in re.finditer(self._VIDEO_INDICATOR, page): + video_id = mobj.group(1) + if video_id not in video_ids: + video_ids.append(video_id) + if len(video_ids) == n: + # Specified n videos reached + for id in video_ids: + self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id]) + return + + if re.search(self._MORE_PAGES_INDICATOR, page) is None: + for id in video_ids: + self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id]) + return + + pagenum = pagenum + 1 + + +class YahooSearchIE(InfoExtractor): + """Information Extractor for Yahoo! Video search queries.""" + _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+' + _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s' + _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"' + _MORE_PAGES_INDICATOR = r'\s*Next' + _max_yahoo_results = 1000 + IE_NAME = u'video.yahoo:search' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_page(self, query, pagenum): + """Report attempt to download playlist page with given number.""" + query = query.decode(preferredencoding()) + self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum)) + + def _real_extract(self, query): + mobj = re.match(self._VALID_URL, query) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) + return + + prefix, query = query.split(':') + prefix = prefix[8:] + query = query.encode('utf-8') + if prefix == '': + self._download_n_results(query, 1) + return + elif prefix == 'all': + self._download_n_results(query, self._max_yahoo_results) + return + else: + try: + n = long(prefix) + if n <= 0: + self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) + return + elif n > self._max_yahoo_results: + self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n)) + n = self._max_yahoo_results + self._download_n_results(query, n) + return + except ValueError: # parsing prefix as integer fails + self._download_n_results(query, 1) + return + + def _download_n_results(self, query, n): + """Downloads a specified number of results for a query""" + + video_ids = [] + already_seen = set() + pagenum = 1 + + while True: + self.report_download_page(query, pagenum) + result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) + request = urllib2.Request(result_url) + try: + page = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) + return + + # Extract video identifiers + for mobj in re.finditer(self._VIDEO_INDICATOR, page): + video_id = mobj.group(1) + if video_id not in already_seen: + video_ids.append(video_id) + already_seen.add(video_id) + if len(video_ids) == n: + # Specified n videos reached + for id in video_ids: + self._downloader.download(['http://video.yahoo.com/watch/%s' % id]) + return + + if re.search(self._MORE_PAGES_INDICATOR, page) is None: + for id in video_ids: + self._downloader.download(['http://video.yahoo.com/watch/%s' % id]) + return + + pagenum = pagenum + 1 + + +class YoutubePlaylistIE(InfoExtractor): + """Information Extractor for YouTube playlists.""" + + _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*' + _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en' + _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&list=(PL)?%s&' + _MORE_PAGES_INDICATOR = r'yt-uix-pager-next' + IE_NAME = u'youtube:playlist' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_page(self, playlist_id, pagenum): + """Report attempt to download playlist page with given number.""" + self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum)) + + def _real_extract(self, url): + # Extract playlist id + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid url: %s' % url) + return + + # Single video case + if mobj.group(3) is not None: + self._downloader.download([mobj.group(3)]) + return + + # Download playlist pages + # prefix is 'p' as default for playlists but there are other types that need extra care + playlist_prefix = mobj.group(1) + if playlist_prefix == 'a': + playlist_access = 'artist' + else: + playlist_prefix = 'p' + playlist_access = 'view_play_list' + playlist_id = mobj.group(2) + video_ids = [] + pagenum = 1 + + while True: + self.report_download_page(playlist_id, pagenum) + url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum) + request = urllib2.Request(url) + try: + page = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) + return + + # Extract video identifiers + ids_in_page = [] + for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page): + if mobj.group(1) not in ids_in_page: + ids_in_page.append(mobj.group(1)) + video_ids.extend(ids_in_page) + + if re.search(self._MORE_PAGES_INDICATOR, page) is None: + break + pagenum = pagenum + 1 + + playliststart = self._downloader.params.get('playliststart', 1) - 1 + playlistend = self._downloader.params.get('playlistend', -1) + if playlistend == -1: + video_ids = video_ids[playliststart:] + else: + video_ids = video_ids[playliststart:playlistend] + + for id in video_ids: + self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) + return + + +class YoutubeUserIE(InfoExtractor): + """Information Extractor for YouTube users.""" + + _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)' + _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s' + _GDATA_PAGE_SIZE = 50 + _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d' + _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]' + IE_NAME = u'youtube:user' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_page(self, username, start_index): + """Report attempt to download user page.""" + self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' % + (username, start_index, start_index + self._GDATA_PAGE_SIZE)) + + def _real_extract(self, url): + # Extract username + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid url: %s' % url) + return + + username = mobj.group(1) + + # Download video ids using YouTube Data API. Result size per + # query is limited (currently to 50 videos) so we need to query + # page by page until there are no video ids - it means we got + # all of them. + + video_ids = [] + pagenum = 0 + + while True: + start_index = pagenum * self._GDATA_PAGE_SIZE + 1 + self.report_download_page(username, start_index) + + request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)) + + try: + page = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) + return + + # Extract video identifiers + ids_in_page = [] + + for mobj in re.finditer(self._VIDEO_INDICATOR, page): + if mobj.group(1) not in ids_in_page: + ids_in_page.append(mobj.group(1)) + + video_ids.extend(ids_in_page) + + # A little optimization - if current page is not + # "full", ie. does not contain PAGE_SIZE video ids then + # we can assume that this page is the last one - there + # are no more ids on further pages - no need to query + # again. + + if len(ids_in_page) < self._GDATA_PAGE_SIZE: + break + + pagenum += 1 + + all_ids_count = len(video_ids) + playliststart = self._downloader.params.get('playliststart', 1) - 1 + playlistend = self._downloader.params.get('playlistend', -1) + + if playlistend == -1: + video_ids = video_ids[playliststart:] + else: + video_ids = video_ids[playliststart:playlistend] + + self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" % + (username, all_ids_count, len(video_ids))) + + for video_id in video_ids: + self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id]) + + +class BlipTVUserIE(InfoExtractor): + """Information Extractor for blip.tv users.""" + + _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$' + _PAGE_SIZE = 12 + IE_NAME = u'blip.tv:user' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_page(self, username, pagenum): + """Report attempt to download user page.""" + self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' % + (self.IE_NAME, username, pagenum)) + + def _real_extract(self, url): + # Extract username + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid url: %s' % url) + return + + username = mobj.group(1) + + page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1' + + request = urllib2.Request(url) + + try: + page = urllib2.urlopen(request).read().decode('utf-8') + mobj = re.search(r'data-users-id="([^"]+)"', page) + page_base = page_base % mobj.group(1) + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) + return + + + # Download video ids using BlipTV Ajax calls. Result size per + # query is limited (currently to 12 videos) so we need to query + # page by page until there are no video ids - it means we got + # all of them. + + video_ids = [] + pagenum = 1 + + while True: + self.report_download_page(username, pagenum) + + request = urllib2.Request( page_base + "&page=" + str(pagenum) ) + + try: + page = urllib2.urlopen(request).read().decode('utf-8') + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) + return + + # Extract video identifiers + ids_in_page = [] + + for mobj in re.finditer(r'href="/([^"]+)"', page): + if mobj.group(1) not in ids_in_page: + ids_in_page.append(unescapeHTML(mobj.group(1))) + + video_ids.extend(ids_in_page) + + # A little optimization - if current page is not + # "full", ie. does not contain PAGE_SIZE video ids then + # we can assume that this page is the last one - there + # are no more ids on further pages - no need to query + # again. + + if len(ids_in_page) < self._PAGE_SIZE: + break + + pagenum += 1 + + all_ids_count = len(video_ids) + playliststart = self._downloader.params.get('playliststart', 1) - 1 + playlistend = self._downloader.params.get('playlistend', -1) + + if playlistend == -1: + video_ids = video_ids[playliststart:] + else: + video_ids = video_ids[playliststart:playlistend] + + self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" % + (self.IE_NAME, username, all_ids_count, len(video_ids))) + + for video_id in video_ids: + self._downloader.download([u'http://blip.tv/'+video_id]) + + +class DepositFilesIE(InfoExtractor): + """Information extractor for depositfiles.com""" + + _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)' + IE_NAME = u'DepositFiles' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_webpage(self, file_id): + """Report webpage download.""" + self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id) + + def report_extraction(self, file_id): + """Report information extraction.""" + self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id) + + def _real_extract(self, url): + file_id = url.split('/')[-1] + # Rebuild url in english locale + url = 'http://depositfiles.com/en/files/' + file_id + + # Retrieve file webpage with 'Free download' button pressed + free_download_indication = { 'gateway_result' : '1' } + request = urllib2.Request(url, urllib.urlencode(free_download_indication)) + try: + self.report_download_webpage(file_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err)) + return + + # Search for the real file URL + mobj = re.search(r'
(Attention.*?)', webpage, re.DOTALL) + if (mobj is not None) and (mobj.group(1) is not None): + restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip() + self._downloader.trouble(u'ERROR: %s' % restriction_message) + else: + self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url) + return + + file_url = mobj.group(1) + file_extension = os.path.splitext(file_url)[1][1:] + + # Search for file title + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + file_title = mobj.group(1).decode('utf-8') + + return [{ + 'id': file_id.decode('utf-8'), + 'url': file_url.decode('utf-8'), + 'uploader': u'NA', + 'upload_date': u'NA', + 'title': file_title, + 'ext': file_extension.decode('utf-8'), + 'format': u'NA', + 'player_url': None, + }] + + +class FacebookIE(InfoExtractor): + """Information Extractor for Facebook""" + + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P\d+)(?:.*)' + _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&' + _NETRC_MACHINE = 'facebook' + _available_formats = ['video', 'highqual', 'lowqual'] + _video_extensions = { + 'video': 'mp4', + 'highqual': 'mp4', + 'lowqual': 'mp4', + } + IE_NAME = u'facebook' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def _reporter(self, message): + """Add header and report message.""" + self._downloader.to_screen(u'[facebook] %s' % message) + + def report_login(self): + """Report attempt to log in.""" + self._reporter(u'Logging in') + + def report_video_webpage_download(self, video_id): + """Report attempt to download video webpage.""" + self._reporter(u'%s: Downloading video webpage' % video_id) + + def report_information_extraction(self, video_id): + """Report attempt to extract video information.""" + self._reporter(u'%s: Extracting video information' % video_id) + + def _parse_page(self, video_webpage): + """Extract video information from page""" + # General data + data = {'title': r'\("video_title", "(.*?)"\)', + 'description': r'
(.*?)
', + 'owner': r'\("video_owner_name", "(.*?)"\)', + 'thumbnail': r'\("thumb_url", "(?P.*?)"\)', + } + video_info = {} + for piece in data.keys(): + mobj = re.search(data[piece], video_webpage) + if mobj is not None: + video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape")) + + # Video urls + video_urls = {} + for fmt in self._available_formats: + mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage) + if mobj is not None: + # URL is in a Javascript segment inside an escaped Unicode format within + # the generally utf-8 page + video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape")) + video_info['video_urls'] = video_urls + + return video_info + + def _real_initialize(self): + if self._downloader is None: + return + + useremail = None + password = None + downloader_params = self._downloader.params + + # Attempt to use provided username and password or .netrc data + if downloader_params.get('username', None) is not None: + useremail = downloader_params['username'] + password = downloader_params['password'] + elif downloader_params.get('usenetrc', False): + try: + info = netrc.netrc().authenticators(self._NETRC_MACHINE) + if info is not None: + useremail = info[0] + password = info[2] + else: + raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) + except (IOError, netrc.NetrcParseError), err: + self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err)) + return + + if useremail is None: + return + + # Log in + login_form = { + 'email': useremail, + 'pass': password, + 'login': 'Log+In' + } + request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form)) + try: + self.report_login() + login_results = urllib2.urlopen(request).read() + if re.search(r'', login_results) is not None: + self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.') + return + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err)) + return + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + video_id = mobj.group('ID') + + # Get video webpage + self.report_video_webpage_download(video_id) + request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id) + try: + page = urllib2.urlopen(request) + video_webpage = page.read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) + return + + # Start extracting information + self.report_information_extraction(video_id) + + # Extract information + video_info = self._parse_page(video_webpage) + + # uploader + if 'owner' not in video_info: + self._downloader.trouble(u'ERROR: unable to extract uploader nickname') + return + video_uploader = video_info['owner'] + + # title + if 'title' not in video_info: + self._downloader.trouble(u'ERROR: unable to extract video title') + return + video_title = video_info['title'] + video_title = video_title.decode('utf-8') + + # thumbnail image + if 'thumbnail' not in video_info: + self._downloader.trouble(u'WARNING: unable to extract video thumbnail') + video_thumbnail = '' + else: + video_thumbnail = video_info['thumbnail'] + + # upload date + upload_date = u'NA' + if 'upload_date' in video_info: + upload_time = video_info['upload_date'] + timetuple = email.utils.parsedate_tz(upload_time) + if timetuple is not None: + try: + upload_date = time.strftime('%Y%m%d', timetuple[0:9]) + except: + pass + + # description + video_description = video_info.get('description', 'No description available.') + + url_map = video_info['video_urls'] + if len(url_map.keys()) > 0: + # Decide which formats to download + req_format = self._downloader.params.get('format', None) + format_limit = self._downloader.params.get('format_limit', None) + + if format_limit is not None and format_limit in self._available_formats: + format_list = self._available_formats[self._available_formats.index(format_limit):] + else: + format_list = self._available_formats + existing_formats = [x for x in format_list if x in url_map] + if len(existing_formats) == 0: + self._downloader.trouble(u'ERROR: no known formats available for video') + return + if req_format is None: + video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality + elif req_format == 'worst': + video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality + elif req_format == '-1': + video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats + else: + # Specific format + if req_format not in url_map: + self._downloader.trouble(u'ERROR: requested format not available') + return + video_url_list = [(req_format, url_map[req_format])] # Specific format + + results = [] + for format_param, video_real_url in video_url_list: + # Extension + video_extension = self._video_extensions.get(format_param, 'mp4') + + results.append({ + 'id': video_id.decode('utf-8'), + 'url': video_real_url.decode('utf-8'), + 'uploader': video_uploader.decode('utf-8'), + 'upload_date': upload_date, + 'title': video_title, + 'ext': video_extension.decode('utf-8'), + 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), + 'thumbnail': video_thumbnail.decode('utf-8'), + 'description': video_description.decode('utf-8'), + 'player_url': None, + }) + return results + +class BlipTVIE(InfoExtractor): + """Information extractor for blip.tv""" + + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$' + _URL_EXT = r'^.*\.([a-z0-9]+)$' + IE_NAME = u'blip.tv' + + def report_extraction(self, file_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id)) + + def report_direct_download(self, title): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title)) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + if '?' in url: + cchar = '&' + else: + cchar = '?' + json_url = url + cchar + 'skin=json&version=2&no_wrap=1' + request = urllib2.Request(json_url.encode('utf-8')) + self.report_extraction(mobj.group(1)) + info = None + try: + urlh = urllib2.urlopen(request) + if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download + basename = url.split('/')[-1] + title,ext = os.path.splitext(basename) + title = title.decode('UTF-8') + ext = ext.replace('.', '') + self.report_direct_download(title) + info = { + 'id': title, + 'url': url, + 'title': title, + 'ext': ext, + 'urlhandle': urlh + } + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err)) + return + if info is None: # Regular URL + try: + json_code = urlh.read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % str(err)) + return + + try: + json_data = json.loads(json_code) + if 'Post' in json_data: + data = json_data['Post'] + else: + data = json_data + + upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d') + video_url = data['media']['url'] + umobj = re.match(self._URL_EXT, video_url) + if umobj is None: + raise ValueError('Can not determine filename extension') + ext = umobj.group(1) + + info = { + 'id': data['item_id'], + 'url': video_url, + 'uploader': data['display_name'], + 'upload_date': upload_date, + 'title': data['title'], + 'ext': ext, + 'format': data['media']['mimeType'], + 'thumbnail': data['thumbnailUrl'], + 'description': data['description'], + 'player_url': data['embedUrl'] + } + except (ValueError,KeyError), err: + self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err)) + return + + std_headers['User-Agent'] = 'iTunes/10.6.1' + return [info] + + +class MyVideoIE(InfoExtractor): + """Information Extractor for myvideo.de.""" + + _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*' + IE_NAME = u'myvideo' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_webpage(self, video_id): + """Report webpage download.""" + self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id) + + def _real_extract(self,url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._download.trouble(u'ERROR: invalid URL: %s' % url) + return + + video_id = mobj.group(1) + + # Get video webpage + request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id) + try: + self.report_download_webpage(video_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) + return + + self.report_extraction(video_id) + mobj = re.search(r'', + webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract media URL') + return + video_url = mobj.group(1) + ('/%s.flv' % video_id) + + mobj = re.search('([^<]+)', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + + video_title = mobj.group(1) + + return [{ + 'id': video_id, + 'url': video_url, + 'uploader': u'NA', + 'upload_date': u'NA', + 'title': video_title, + 'ext': u'flv', + 'format': u'NA', + 'player_url': None, + }] + +class ComedyCentralIE(InfoExtractor): + """Information extractor for The Daily Show and Colbert Report """ + + _VALID_URL = r'^(:(?Ptds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?Pthedailyshow|colbertnation)\.com/full-episodes/(?P.*)$' + IE_NAME = u'comedycentral' + + def report_extraction(self, episode_id): + self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id) + + def report_config_download(self, episode_id): + self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id) + + def report_index_download(self, episode_id): + self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id) + + def report_player_url(self, episode_id): + self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + if mobj.group('shortname'): + if mobj.group('shortname') in ('tds', 'thedailyshow'): + url = u'http://www.thedailyshow.com/full-episodes/' + else: + url = u'http://www.colbertnation.com/full-episodes/' + mobj = re.match(self._VALID_URL, url) + assert mobj is not None + + dlNewest = not mobj.group('episode') + if dlNewest: + epTitle = mobj.group('showname') + else: + epTitle = mobj.group('episode') + + req = urllib2.Request(url) + self.report_extraction(epTitle) + try: + htmlHandle = urllib2.urlopen(req) + html = htmlHandle.read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err)) + return + if dlNewest: + url = htmlHandle.geturl() + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url) + return + if mobj.group('episode') == '': + self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url) + return + epTitle = mobj.group('episode') + + mMovieParams = re.findall('(?:[^/]+)/(?P[^/?]+)[/?]?.*$' + IE_NAME = u'escapist' + + def report_extraction(self, showName): + self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName) + + def report_config_download(self, showName): + self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + showName = mobj.group('showname') + videoId = mobj.group('episode') + + self.report_extraction(showName) + try: + webPage = urllib2.urlopen(url) + webPageBytes = webPage.read() + m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type']) + webPage = webPageBytes.decode(m.group(1) if m else 'utf-8') + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err)) + return + + descMatch = re.search('[0-9]+)/(?P.*)$' + IE_NAME = u'collegehumor' + + def report_webpage(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + video_id = mobj.group('videoid') + + self.report_webpage(video_id) + request = urllib2.Request(url) + try: + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) + return + + m = re.search(r'id="video:(?P[0-9]+)"', webpage) + if m is None: + self._downloader.trouble(u'ERROR: Cannot extract internal video ID') + return + internal_video_id = m.group('internalvideoid') + + info = { + 'id': video_id, + 'internal_id': internal_video_id, + } + + self.report_extraction(video_id) + xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id + try: + metaXml = urllib2.urlopen(xmlUrl).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err)) + return + + mdoc = xml.etree.ElementTree.fromstring(metaXml) + try: + videoNode = mdoc.findall('./video')[0] + info['description'] = videoNode.findall('./description')[0].text + info['title'] = videoNode.findall('./caption')[0].text + info['url'] = videoNode.findall('./file')[0].text + info['thumbnail'] = videoNode.findall('./thumbnail')[0].text + info['ext'] = info['url'].rpartition('.')[2] + info['format'] = info['ext'] + except IndexError: + self._downloader.trouble(u'\nERROR: Invalid metadata XML file') + return + + return [info] + + +class XVideosIE(InfoExtractor): + """Information extractor for xvideos.com""" + + _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)' + IE_NAME = u'xvideos' + + def report_webpage(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + video_id = mobj.group(1).decode('utf-8') + + self.report_webpage(video_id) + + request = urllib2.Request(r'http://www.xvideos.com/video' + video_id) + try: + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) + return + + self.report_extraction(video_id) + + + # Extract video URL + mobj = re.search(r'flv_url=(.+?)&', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video url') + return + video_url = urllib2.unquote(mobj.group(1).decode('utf-8')) + + + # Extract title + mobj = re.search(r'(.*?)\s+-\s+XVID', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video title') + return + video_title = mobj.group(1).decode('utf-8') + + + # Extract video thumbnail + mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video thumbnail') + return + video_thumbnail = mobj.group(0).decode('utf-8') + + info = { + 'id': video_id, + 'url': video_url, + 'uploader': None, + 'upload_date': None, + 'title': video_title, + 'ext': 'flv', + 'format': 'flv', + 'thumbnail': video_thumbnail, + 'description': None, + 'player_url': None, + } + + return [info] + + +class SoundcloudIE(InfoExtractor): + """Information extractor for soundcloud.com + To access the media, the uid of the song and a stream token + must be extracted from the page source and the script must make + a request to media.soundcloud.com/crossdomain.xml. Then + the media can be grabbed by requesting from an url composed + of the stream token and uid + """ + + _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)' + IE_NAME = u'soundcloud' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_webpage(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + # extract uploader (which is in the url) + uploader = mobj.group(1).decode('utf-8') + # extract simple title (uploader + slug of song title) + slug_title = mobj.group(2).decode('utf-8') + simple_title = uploader + u'-' + slug_title + + self.report_webpage('%s/%s' % (uploader, slug_title)) + + request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title)) + try: + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) + return + + self.report_extraction('%s/%s' % (uploader, slug_title)) + + # extract uid and stream token that soundcloud hands out for access + mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage) + if mobj: + video_id = mobj.group(1) + stream_token = mobj.group(2) + + # extract unsimplified title + mobj = re.search('"title":"(.*?)",', webpage) + if mobj: + title = mobj.group(1).decode('utf-8') + else: + title = simple_title + + # construct media url (with uid/token) + mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s" + mediaURL = mediaURL % (video_id, stream_token) + + # description + description = u'No description available' + mobj = re.search('track-description-value"><p>(.*?)</p>', webpage) + if mobj: + description = mobj.group(1) + + # upload date + upload_date = None + mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage) + if mobj: + try: + upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d') + except Exception, e: + self._downloader.to_stderr(str(e)) + + # for soundcloud, a request to a cross domain is required for cookies + request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers) + + return [{ + 'id': video_id.decode('utf-8'), + 'url': mediaURL, + 'uploader': uploader.decode('utf-8'), + 'upload_date': upload_date, + 'title': title, + 'ext': u'mp3', + 'format': u'NA', + 'player_url': None, + 'description': description.decode('utf-8') + }] + + +class InfoQIE(InfoExtractor): + """Information extractor for infoq.com""" + + _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$' + IE_NAME = u'infoq' + + def report_webpage(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + self.report_webpage(url) + + request = urllib2.Request(url) + try: + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) + return + + self.report_extraction(url) + + + # Extract video URL + mobj = re.search(r"jsclassref='([^']*)'", webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video url') + return + video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64')) + + + # Extract title + mobj = re.search(r'contentTitle = "(.*?)";', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract video title') + return + video_title = mobj.group(1).decode('utf-8') + + # Extract description + video_description = u'No description available.' + mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage) + if mobj is not None: + video_description = mobj.group(1).decode('utf-8') + + video_filename = video_url.split('/')[-1] + video_id, extension = video_filename.split('.') + + info = { + 'id': video_id, + 'url': video_url, + 'uploader': None, + 'upload_date': None, + 'title': video_title, + 'ext': extension, + 'format': extension, # Extension is always(?) mp4, but seems to be flv + 'thumbnail': None, + 'description': video_description, + 'player_url': None, + } + + return [info] + +class MixcloudIE(InfoExtractor): + """Information extractor for www.mixcloud.com""" + _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)' + IE_NAME = u'mixcloud' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + def report_download_json(self, file_id): + """Report JSON download.""" + self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME) + + def report_extraction(self, file_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id)) + + def get_urls(self, jsonData, fmt, bitrate='best'): + """Get urls from 'audio_formats' section in json""" + file_url = None + try: + bitrate_list = jsonData[fmt] + if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list: + bitrate = max(bitrate_list) # select highest + + url_list = jsonData[fmt][bitrate] + except TypeError: # we have no bitrate info. + url_list = jsonData[fmt] + return url_list + + def check_urls(self, url_list): + """Returns 1st active url from list""" + for url in url_list: + try: + urllib2.urlopen(url) + return url + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + url = None + + return None + + def _print_formats(self, formats): + print 'Available formats:' + for fmt in formats.keys(): + for b in formats[fmt]: + try: + ext = formats[fmt][b][0] + print '%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]) + except TypeError: # we have no bitrate info + ext = formats[fmt][0] + print '%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]) + break + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + # extract uploader & filename from url + uploader = mobj.group(1).decode('utf-8') + file_id = uploader + "-" + mobj.group(2).decode('utf-8') + + # construct API request + file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json' + # retrieve .json file with links to files + request = urllib2.Request(file_url) + try: + self.report_download_json(file_url) + jsonData = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % str(err)) + return + + # parse JSON + json_data = json.loads(jsonData) + player_url = json_data['player_swf_url'] + formats = dict(json_data['audio_formats']) + + req_format = self._downloader.params.get('format', None) + bitrate = None + + if self._downloader.params.get('listformats', None): + self._print_formats(formats) + return + + if req_format is None or req_format == 'best': + for format_param in formats.keys(): + url_list = self.get_urls(formats, format_param) + # check urls + file_url = self.check_urls(url_list) + if file_url is not None: + break # got it! + else: + if req_format not in formats.keys(): + self._downloader.trouble(u'ERROR: format is not available') + return + + url_list = self.get_urls(formats, req_format) + file_url = self.check_urls(url_list) + format_param = req_format + + return [{ + 'id': file_id.decode('utf-8'), + 'url': file_url.decode('utf-8'), + 'uploader': uploader.decode('utf-8'), + 'upload_date': u'NA', + 'title': json_data['name'], + 'ext': file_url.split('.')[-1].decode('utf-8'), + 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), + 'thumbnail': json_data['thumbnail_url'], + 'description': json_data['description'], + 'player_url': player_url.decode('utf-8'), + }] + +class StanfordOpenClassroomIE(InfoExtractor): + """Information extractor for Stanford's Open ClassRoom""" + + _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$' + IE_NAME = u'stanfordoc' + + def report_download_webpage(self, objid): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid)) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + + if mobj.group('course') and mobj.group('video'): # A specific video + course = mobj.group('course') + video = mobj.group('video') + info = { + 'id': course + '_' + video, + } + + self.report_extraction(info['id']) + baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/' + xmlUrl = baseUrl + video + '.xml' + try: + metaXml = urllib2.urlopen(xmlUrl).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err)) + return + mdoc = xml.etree.ElementTree.fromstring(metaXml) + try: + info['title'] = mdoc.findall('./title')[0].text + info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text + except IndexError: + self._downloader.trouble(u'\nERROR: Invalid metadata XML file') + return + info['ext'] = info['url'].rpartition('.')[2] + info['format'] = info['ext'] + return [info] + elif mobj.group('course'): # A course page + course = mobj.group('course') + info = { + 'id': course, + 'type': 'playlist', + } + + self.report_download_webpage(info['id']) + try: + coursepage = urllib2.urlopen(url).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err)) + return + + m = re.search('<h1>([^<]+)</h1>', coursepage) + if m: + info['title'] = unescapeHTML(m.group(1)) + else: + info['title'] = info['id'] + + m = re.search('<description>([^<]+)</description>', coursepage) + if m: + info['description'] = unescapeHTML(m.group(1)) + + links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage)) + info['list'] = [ + { + 'type': 'reference', + 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage), + } + for vpage in links] + results = [] + for entry in info['list']: + assert entry['type'] == 'reference' + results += self.extract(entry['url']) + return results + + else: # Root page + info = { + 'id': 'Stanford OpenClassroom', + 'type': 'playlist', + } + + self.report_download_webpage(info['id']) + rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php' + try: + rootpage = urllib2.urlopen(rootURL).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err)) + return + + info['title'] = info['id'] + + links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage)) + info['list'] = [ + { + 'type': 'reference', + 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage), + } + for cpage in links] + + results = [] + for entry in info['list']: + assert entry['type'] == 'reference' + results += self.extract(entry['url']) + return results + +class MTVIE(InfoExtractor): + """Information extractor for MTV.com""" + + _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$' + IE_NAME = u'mtv' + + def report_webpage(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) + + def report_extraction(self, video_id): + """Report information extraction.""" + self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + self._downloader.trouble(u'ERROR: invalid URL: %s' % url) + return + if not mobj.group('proto'): + url = 'http://' + url + video_id = mobj.group('videoid') + self.report_webpage(video_id) + + request = urllib2.Request(url) + try: + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) + return + + mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract song name') + return + song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1')) + mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract performer') + return + performer = unescapeHTML(mobj.group(1).decode('iso-8859-1')) + video_title = performer + ' - ' + song_name + + mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to mtvn_uri') + return + mtvn_uri = mobj.group(1) + + mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract content id') + return + content_id = mobj.group(1) + + videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri + self.report_extraction(video_id) + request = urllib2.Request(videogen_url) + try: + metadataXml = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % str(err)) + return + + mdoc = xml.etree.ElementTree.fromstring(metadataXml) + renditions = mdoc.findall('.//rendition') + + # For now, always pick the highest quality. + rendition = renditions[-1] + + try: + _,_,ext = rendition.attrib['type'].partition('/') + format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate'] + video_url = rendition.find('./src').text + except KeyError: + self._downloader.trouble('Invalid rendition field.') + return + + info = { + 'id': video_id, + 'url': video_url, + 'uploader': performer, + 'title': video_title, + 'ext': ext, + 'format': format, + } + + return [info] diff --git a/youtube_dl/PostProcessor.py b/youtube_dl/PostProcessor.py new file mode 100644 index 0000000..527dc3a --- /dev/null +++ b/youtube_dl/PostProcessor.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import subprocess +import sys +import time + +from utils import * + + +class PostProcessor(object): + """Post Processor class. + + PostProcessor objects can be added to downloaders with their + add_post_processor() method. When the downloader has finished a + successful download, it will take its internal chain of PostProcessors + and start calling the run() method on each one of them, first with + an initial argument and then with the returned value of the previous + PostProcessor. + + The chain will be stopped if one of them ever returns None or the end + of the chain is reached. + + PostProcessor objects follow a "mutual registration" process similar + to InfoExtractor objects. + """ + + _downloader = None + + def __init__(self, downloader=None): + self._downloader = downloader + + def set_downloader(self, downloader): + """Sets the downloader for this PP.""" + self._downloader = downloader + + def run(self, information): + """Run the PostProcessor. + + The "information" argument is a dictionary like the ones + composed by InfoExtractors. The only difference is that this + one has an extra field called "filepath" that points to the + downloaded file. + + When this method returns None, the postprocessing chain is + stopped. However, this method may return an information + dictionary that will be passed to the next postprocessing + object in the chain. It can be the one it received after + changing some fields. + + In addition, this method may raise a PostProcessingError + exception that will be taken into account by the downloader + it was called from. + """ + return information # by default, do nothing + +class AudioConversionError(BaseException): + def __init__(self, message): + self.message = message + +class FFmpegExtractAudioPP(PostProcessor): + def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False): + PostProcessor.__init__(self, downloader) + if preferredcodec is None: + preferredcodec = 'best' + self._preferredcodec = preferredcodec + self._preferredquality = preferredquality + self._keepvideo = keepvideo + self._exes = self.detect_executables() + + @staticmethod + def detect_executables(): + available = {'avprobe' : False, 'avconv' : False, 'ffmpeg' : False, 'ffprobe' : False} + for path in os.environ["PATH"].split(os.pathsep): + for program in available.keys(): + exe_file = os.path.join(path, program) + if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): + available[program] = exe_file + return available + + def get_audio_codec(self, path): + if not self._exes['ffprobe'] and not self._exes['avprobe']: return None + try: + cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)] + handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE) + output = handle.communicate()[0] + if handle.wait() != 0: + return None + except (IOError, OSError): + return None + audio_codec = None + for line in output.split('\n'): + if line.startswith('codec_name='): + audio_codec = line.split('=')[1].strip() + elif line.strip() == 'codec_type=audio' and audio_codec is not None: + return audio_codec + return None + + def run_ffmpeg(self, path, out_path, codec, more_opts): + if not self._exes['ffmpeg'] and not self._exes['avconv']: + raise AudioConversionError('ffmpeg or avconv not found. Please install one.') + if codec is None: + acodec_opts = [] + else: + acodec_opts = ['-acodec', codec] + cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn'] + + acodec_opts + more_opts + + ['--', encodeFilename(out_path)]) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout,stderr = p.communicate() + if p.returncode != 0: + msg = stderr.strip().split('\n')[-1] + raise AudioConversionError(msg) + + def run(self, information): + path = information['filepath'] + + filecodec = self.get_audio_codec(path) + if filecodec is None: + self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe') + return None + + more_opts = [] + if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'): + if self._preferredcodec == 'm4a' and filecodec == 'aac': + # Lossless, but in another container + acodec = 'copy' + extension = self._preferredcodec + more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc'] + elif filecodec in ['aac', 'mp3', 'vorbis']: + # Lossless if possible + acodec = 'copy' + extension = filecodec + if filecodec == 'aac': + more_opts = ['-f', 'adts'] + if filecodec == 'vorbis': + extension = 'ogg' + else: + # MP3 otherwise. + acodec = 'libmp3lame' + extension = 'mp3' + more_opts = [] + if self._preferredquality is not None: + more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality] + else: + # We convert the audio (lossy) + acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec] + extension = self._preferredcodec + more_opts = [] + if self._preferredquality is not None: + more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality] + if self._preferredcodec == 'aac': + more_opts += ['-f', 'adts'] + if self._preferredcodec == 'm4a': + more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc'] + if self._preferredcodec == 'vorbis': + extension = 'ogg' + if self._preferredcodec == 'wav': + extension = 'wav' + more_opts += ['-f', 'wav'] + + prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups + new_path = prefix + sep + extension + self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path) + try: + self.run_ffmpeg(path, new_path, acodec, more_opts) + except: + etype,e,tb = sys.exc_info() + if isinstance(e, AudioConversionError): + self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message) + else: + self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg')) + return None + + # Try to update the date time for extracted audio file. + if information.get('filetime') is not None: + try: + os.utime(encodeFilename(new_path), (time.time(), information['filetime'])) + except: + self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file') + + if not self._keepvideo: + try: + os.remove(encodeFilename(path)) + except (IOError, OSError): + self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') + return None + + information['filepath'] = new_path + return information diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py old mode 100755 new mode 100644 index 7897240..8695184 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -15,4203 +15,95 @@ __authors__ = ( 'Kevin Ngo', 'Ori Avtalion', 'shizeeg', + 'Filippo Valsorda', ) __license__ = 'Public Domain' __version__ = '2012.02.27' UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl' +UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSION' +UPDATE_URL_EXE = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl.exe' import cookielib -import datetime import getpass -import gzip -import htmlentitydefs -import HTMLParser -import httplib -import locale -import math -import netrc import optparse import os -import os.path import re import shlex import socket -import string import subprocess import sys -import time -import urllib import urllib2 import warnings -import zlib - -if os.name == 'nt': - import ctypes - -try: - import email.utils -except ImportError: # Python 2.4 - import email.Utils -try: - import cStringIO as StringIO -except ImportError: - import StringIO - -# parse_qs was moved from the cgi module to the urlparse module recently. -try: - from urlparse import parse_qs -except ImportError: - from cgi import parse_qs - -try: - import lxml.etree -except ImportError: - pass # Handled below - -try: - import xml.etree.ElementTree -except ImportError: # Python<2.5: Not officially supported, but let it slip - warnings.warn('xml.etree.ElementTree support is missing. Consider upgrading to Python >= 2.5 if you get related errors.') - -std_headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1', - 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Encoding': 'gzip, deflate', - 'Accept-Language': 'en-us,en;q=0.5', -} - -try: - import json -except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson): - import re - class json(object): - @staticmethod - def loads(s): - s = s.decode('UTF-8') - def raiseError(msg, i): - raise ValueError(msg + ' at position ' + str(i) + ' of ' + repr(s) + ': ' + repr(s[i:])) - def skipSpace(i, expectMore=True): - while i < len(s) and s[i] in ' \t\r\n': - i += 1 - if expectMore: - if i >= len(s): - raiseError('Premature end', i) - return i - def decodeEscape(match): - esc = match.group(1) - _STATIC = { - '"': '"', - '\\': '\\', - '/': '/', - 'b': unichr(0x8), - 'f': unichr(0xc), - 'n': '\n', - 'r': '\r', - 't': '\t', - } - if esc in _STATIC: - return _STATIC[esc] - if esc[0] == 'u': - if len(esc) == 1+4: - return unichr(int(esc[1:5], 16)) - if len(esc) == 5+6 and esc[5:7] == '\\u': - hi = int(esc[1:5], 16) - low = int(esc[7:11], 16) - return unichr((hi - 0xd800) * 0x400 + low - 0xdc00 + 0x10000) - raise ValueError('Unknown escape ' + str(esc)) - def parseString(i): - i += 1 - e = i - while True: - e = s.index('"', e) - bslashes = 0 - while s[e-bslashes-1] == '\\': - bslashes += 1 - if bslashes % 2 == 1: - e += 1 - continue - break - rexp = re.compile(r'\\(u[dD][89aAbB][0-9a-fA-F]{2}\\u[0-9a-fA-F]{4}|u[0-9a-fA-F]{4}|.|$)') - stri = rexp.sub(decodeEscape, s[i:e]) - return (e+1,stri) - def parseObj(i): - i += 1 - res = {} - i = skipSpace(i) - if s[i] == '}': # Empty dictionary - return (i+1,res) - while True: - if s[i] != '"': - raiseError('Expected a string object key', i) - i,key = parseString(i) - i = skipSpace(i) - if i >= len(s) or s[i] != ':': - raiseError('Expected a colon', i) - i,val = parse(i+1) - res[key] = val - i = skipSpace(i) - if s[i] == '}': - return (i+1, res) - if s[i] != ',': - raiseError('Expected comma or closing curly brace', i) - i = skipSpace(i+1) - def parseArray(i): - res = [] - i = skipSpace(i+1) - if s[i] == ']': # Empty array - return (i+1,res) - while True: - i,val = parse(i) - res.append(val) - i = skipSpace(i) # Raise exception if premature end - if s[i] == ']': - return (i+1, res) - if s[i] != ',': - raiseError('Expected a comma or closing bracket', i) - i = skipSpace(i+1) - def parseDiscrete(i): - for k,v in {'true': True, 'false': False, 'null': None}.items(): - if s.startswith(k, i): - return (i+len(k), v) - raiseError('Not a boolean (or null)', i) - def parseNumber(i): - mobj = re.match('^(-?(0|[1-9][0-9]*)(\.[0-9]*)?([eE][+-]?[0-9]+)?)', s[i:]) - if mobj is None: - raiseError('Not a number', i) - nums = mobj.group(1) - if '.' in nums or 'e' in nums or 'E' in nums: - return (i+len(nums), float(nums)) - return (i+len(nums), int(nums)) - CHARMAP = {'{': parseObj, '[': parseArray, '"': parseString, 't': parseDiscrete, 'f': parseDiscrete, 'n': parseDiscrete} - def parse(i): - i = skipSpace(i) - i,res = CHARMAP.get(s[i], parseNumber)(i) - i = skipSpace(i, False) - return (i,res) - i,res = parse(0) - if i < len(s): - raise ValueError('Extra data at end of input (index ' + str(i) + ' of ' + repr(s) + ': ' + repr(s[i:]) + ')') - return res - -def preferredencoding(): - """Get preferred encoding. - - Returns the best encoding scheme for the system, based on - locale.getpreferredencoding() and some further tweaks. - """ - def yield_preferredencoding(): - try: - pref = locale.getpreferredencoding() - u'TEST'.encode(pref) - except: - pref = 'UTF-8' - while True: - yield pref - return yield_preferredencoding().next() - - -def htmlentity_transform(matchobj): - """Transforms an HTML entity to a Unicode character. - - This function receives a match object and is intended to be used with - the re.sub() function. - """ - entity = matchobj.group(1) - - # Known non-numeric HTML entity - if entity in htmlentitydefs.name2codepoint: - return unichr(htmlentitydefs.name2codepoint[entity]) - - # Unicode character - mobj = re.match(ur'(?u)#(x?\d+)', entity) - if mobj is not None: - numstr = mobj.group(1) - if numstr.startswith(u'x'): - base = 16 - numstr = u'0%s' % numstr - else: - base = 10 - return unichr(long(numstr, base)) - - # Unknown entity in name, return its literal representation - return (u'&%s;' % entity) - - -def sanitize_title(utitle): - """Sanitizes a video title so it could be used as part of a filename.""" - utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle) - return utitle.replace(unicode(os.sep), u'%') - - -def sanitize_open(filename, open_mode): - """Try to open the given filename, and slightly tweak it if this fails. - - Attempts to open the given filename. If this fails, it tries to change - the filename slightly, step by step, until it's either able to open it - or it fails and raises a final exception, like the standard open() - function. - - It returns the tuple (stream, definitive_file_name). - """ - try: - if filename == u'-': - if sys.platform == 'win32': - import msvcrt - msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - return (sys.stdout, filename) - stream = open(_encodeFilename(filename), open_mode) - return (stream, filename) - except (IOError, OSError), err: - # In case of error, try to remove win32 forbidden chars - filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename) - - # An exception here should be caught in the caller - stream = open(_encodeFilename(filename), open_mode) - return (stream, filename) - - -def timeconvert(timestr): - """Convert RFC 2822 defined time string into system timestamp""" - timestamp = None - timetuple = email.utils.parsedate_tz(timestr) - if timetuple is not None: - timestamp = email.utils.mktime_tz(timetuple) - return timestamp - -def _simplify_title(title): - expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE) - return expr.sub(u'_', title).strip(u'_') - -def _orderedSet(iterable): - """ Remove all duplicates from the input iterable """ - res = [] - for el in iterable: - if el not in res: - res.append(el) - return res - -def _unescapeHTML(s): - """ - @param s a string (of type unicode) - """ - assert type(s) == type(u'') - - htmlParser = HTMLParser.HTMLParser() - return htmlParser.unescape(s) - -def _encodeFilename(s): - """ - @param s The name of the file (of type unicode) - """ - - assert type(s) == type(u'') - - if sys.platform == 'win32' and sys.getwindowsversion().major >= 5: - # Pass u'' directly to use Unicode APIs on Windows 2000 and up - # (Detecting Windows NT 4 is tricky because 'major >= 4' would - # match Windows 9x series as well. Besides, NT 4 is obsolete.) - return s - else: - return s.encode(sys.getfilesystemencoding(), 'ignore') - -class DownloadError(Exception): - """Download Error exception. - - This exception may be thrown by FileDownloader objects if they are not - configured to continue on errors. They will contain the appropriate - error message. - """ - pass - - -class SameFileError(Exception): - """Same File exception. - - This exception will be thrown by FileDownloader objects if they detect - multiple files would have to be downloaded to the same file on disk. - """ - pass - - -class PostProcessingError(Exception): - """Post Processing exception. - - This exception may be raised by PostProcessor's .run() method to - indicate an error in the postprocessing task. - """ - pass - -class MaxDownloadsReached(Exception): - """ --max-downloads limit has been reached. """ - pass - - -class UnavailableVideoError(Exception): - """Unavailable Format exception. - - This exception will be thrown when a video is requested - in a format that is not available for that video. - """ - pass - - -class ContentTooShortError(Exception): - """Content Too Short exception. - - This exception may be raised by FileDownloader objects when a file they - download is too small for what the server announced first, indicating - the connection was probably interrupted. - """ - # Both in bytes - downloaded = None - expected = None - - def __init__(self, downloaded, expected): - self.downloaded = downloaded - self.expected = expected - - -class YoutubeDLHandler(urllib2.HTTPHandler): - """Handler for HTTP requests and responses. - - This class, when installed with an OpenerDirector, automatically adds - the standard headers to every HTTP request and handles gzipped and - deflated responses from web servers. If compression is to be avoided in - a particular request, the original request in the program code only has - to include the HTTP header "Youtubedl-No-Compression", which will be - removed before making the real request. - - Part of this code was copied from: - - http://techknack.net/python-urllib2-handlers/ - - Andrew Rowls, the author of that code, agreed to release it to the - public domain. - """ - - @staticmethod - def deflate(data): - try: - return zlib.decompress(data, -zlib.MAX_WBITS) - except zlib.error: - return zlib.decompress(data) - - @staticmethod - def addinfourl_wrapper(stream, headers, url, code): - if hasattr(urllib2.addinfourl, 'getcode'): - return urllib2.addinfourl(stream, headers, url, code) - ret = urllib2.addinfourl(stream, headers, url) - ret.code = code - return ret - - def http_request(self, req): - for h in std_headers: - if h in req.headers: - del req.headers[h] - req.add_header(h, std_headers[h]) - if 'Youtubedl-no-compression' in req.headers: - if 'Accept-encoding' in req.headers: - del req.headers['Accept-encoding'] - del req.headers['Youtubedl-no-compression'] - return req - - def http_response(self, req, resp): - old_resp = resp - # gzip - if resp.headers.get('Content-encoding', '') == 'gzip': - gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r') - resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) - resp.msg = old_resp.msg - # deflate - if resp.headers.get('Content-encoding', '') == 'deflate': - gz = StringIO.StringIO(self.deflate(resp.read())) - resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) - resp.msg = old_resp.msg - return resp - - -class FileDownloader(object): - """File Downloader class. - - File downloader objects are the ones responsible of downloading the - actual video file and writing it to disk if the user has requested - it, among some other tasks. In most cases there should be one per - program. As, given a video URL, the downloader doesn't know how to - extract all the needed information, task that InfoExtractors do, it - has to pass the URL to one of them. - - For this, file downloader objects have a method that allows - InfoExtractors to be registered in a given order. When it is passed - a URL, the file downloader handles it to the first InfoExtractor it - finds that reports being able to handle it. The InfoExtractor extracts - all the information about the video or videos the URL refers to, and - asks the FileDownloader to process the video information, possibly - downloading the video. - - File downloaders accept a lot of parameters. In order not to saturate - the object constructor with arguments, it receives a dictionary of - options instead. These options are available through the params - attribute for the InfoExtractors to use. The FileDownloader also - registers itself as the downloader in charge for the InfoExtractors - that are added to it, so this is a "mutual registration". - - Available options: - - username: Username for authentication purposes. - password: Password for authentication purposes. - usenetrc: Use netrc for authentication instead. - quiet: Do not print messages to stdout. - forceurl: Force printing final URL. - forcetitle: Force printing title. - forcethumbnail: Force printing thumbnail URL. - forcedescription: Force printing description. - forcefilename: Force printing final filename. - simulate: Do not download the video files. - format: Video format code. - format_limit: Highest quality format to try. - outtmpl: Template for output names. - ignoreerrors: Do not stop on download errors. - ratelimit: Download speed limit, in bytes/sec. - nooverwrites: Prevent overwriting files. - retries: Number of times to retry for HTTP error 5xx - continuedl: Try to continue downloads if possible. - noprogress: Do not print the progress bar. - playliststart: Playlist item to start at. - playlistend: Playlist item to end at. - matchtitle: Download only matching titles. - rejecttitle: Reject downloads for matching titles. - logtostderr: Log messages to stderr instead of stdout. - consoletitle: Display progress in console window's titlebar. - nopart: Do not use temporary .part files. - updatetime: Use the Last-modified header to set output file timestamps. - writedescription: Write the video description to a .description file - writeinfojson: Write the video description to a .info.json file - """ - - params = None - _ies = [] - _pps = [] - _download_retcode = None - _num_downloads = None - _screen_file = None - - def __init__(self, params): - """Create a FileDownloader object with the given options.""" - self._ies = [] - self._pps = [] - self._download_retcode = 0 - self._num_downloads = 0 - self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] - self.params = params - - @staticmethod - def format_bytes(bytes): - if bytes is None: - return 'N/A' - if type(bytes) is str: - bytes = float(bytes) - if bytes == 0.0: - exponent = 0 - else: - exponent = long(math.log(bytes, 1024.0)) - suffix = 'bkMGTPEZY'[exponent] - converted = float(bytes) / float(1024 ** exponent) - return '%.2f%s' % (converted, suffix) - - @staticmethod - def calc_percent(byte_counter, data_len): - if data_len is None: - return '---.-%' - return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0)) - - @staticmethod - def calc_eta(start, now, total, current): - if total is None: - return '--:--' - dif = now - start - if current == 0 or dif < 0.001: # One millisecond - return '--:--' - rate = float(current) / dif - eta = long((float(total) - float(current)) / rate) - (eta_mins, eta_secs) = divmod(eta, 60) - if eta_mins > 99: - return '--:--' - return '%02d:%02d' % (eta_mins, eta_secs) - - @staticmethod - def calc_speed(start, now, bytes): - dif = now - start - if bytes == 0 or dif < 0.001: # One millisecond - return '%10s' % '---b/s' - return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif)) - - @staticmethod - def best_block_size(elapsed_time, bytes): - new_min = max(bytes / 2.0, 1.0) - new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB - if elapsed_time < 0.001: - return long(new_max) - rate = bytes / elapsed_time - if rate > new_max: - return long(new_max) - if rate < new_min: - return long(new_min) - return long(rate) - - @staticmethod - def parse_bytes(bytestr): - """Parse a string indicating a byte quantity into a long integer.""" - matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) - if matchobj is None: - return None - number = float(matchobj.group(1)) - multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) - return long(round(number * multiplier)) - - def add_info_extractor(self, ie): - """Add an InfoExtractor object to the end of the list.""" - self._ies.append(ie) - ie.set_downloader(self) - - def add_post_processor(self, pp): - """Add a PostProcessor object to the end of the chain.""" - self._pps.append(pp) - pp.set_downloader(self) - - def to_screen(self, message, skip_eol=False): - """Print message to stdout if not in quiet mode.""" - assert type(message) == type(u'') - if not self.params.get('quiet', False): - terminator = [u'\n', u''][skip_eol] - output = message + terminator - - if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr - output = output.encode(preferredencoding(), 'ignore') - self._screen_file.write(output) - self._screen_file.flush() - - def to_stderr(self, message): - """Print message to stderr.""" - print >>sys.stderr, message.encode(preferredencoding()) - - def to_cons_title(self, message): - """Set console/terminal window title to message.""" - if not self.params.get('consoletitle', False): - return - if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): - # c_wchar_p() might not be necessary if `message` is - # already of type unicode() - ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) - elif 'TERM' in os.environ: - sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding())) - - def fixed_template(self): - """Checks if the output template is fixed.""" - return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None) - - def trouble(self, message=None): - """Determine action to take when a download problem appears. - - Depending on if the downloader has been configured to ignore - download errors or not, this method may throw an exception or - not when errors are found, after printing the message. - """ - if message is not None: - self.to_stderr(message) - if not self.params.get('ignoreerrors', False): - raise DownloadError(message) - self._download_retcode = 1 - - def slow_down(self, start_time, byte_counter): - """Sleep if the download speed is over the rate limit.""" - rate_limit = self.params.get('ratelimit', None) - if rate_limit is None or byte_counter == 0: - return - now = time.time() - elapsed = now - start_time - if elapsed <= 0.0: - return - speed = float(byte_counter) / elapsed - if speed > rate_limit: - time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) - - def temp_name(self, filename): - """Returns a temporary filename for the given filename.""" - if self.params.get('nopart', False) or filename == u'-' or \ - (os.path.exists(_encodeFilename(filename)) and not os.path.isfile(_encodeFilename(filename))): - return filename - return filename + u'.part' - - def undo_temp_name(self, filename): - if filename.endswith(u'.part'): - return filename[:-len(u'.part')] - return filename - - def try_rename(self, old_filename, new_filename): - try: - if old_filename == new_filename: - return - os.rename(_encodeFilename(old_filename), _encodeFilename(new_filename)) - except (IOError, OSError), err: - self.trouble(u'ERROR: unable to rename file') - - def try_utime(self, filename, last_modified_hdr): - """Try to set the last-modified time of the given file.""" - if last_modified_hdr is None: - return - if not os.path.isfile(_encodeFilename(filename)): - return - timestr = last_modified_hdr - if timestr is None: - return - filetime = timeconvert(timestr) - if filetime is None: - return filetime - try: - os.utime(filename, (time.time(), filetime)) - except: - pass - return filetime - - def report_writedescription(self, descfn): - """ Report that the description file is being written """ - self.to_screen(u'[info] Writing video description to: ' + descfn) - - def report_writeinfojson(self, infofn): - """ Report that the metadata file has been written """ - self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) - - def report_destination(self, filename): - """Report destination filename.""" - self.to_screen(u'[download] Destination: ' + filename) - - def report_progress(self, percent_str, data_len_str, speed_str, eta_str): - """Report download progress.""" - if self.params.get('noprogress', False): - return - self.to_screen(u'\r[download] %s of %s at %s ETA %s' % - (percent_str, data_len_str, speed_str, eta_str), skip_eol=True) - self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' % - (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) - - def report_resuming_byte(self, resume_len): - """Report attempt to resume at given byte.""" - self.to_screen(u'[download] Resuming download at byte %s' % resume_len) - - def report_retry(self, count, retries): - """Report retry in case of HTTP error 5xx""" - self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) - - def report_file_already_downloaded(self, file_name): - """Report file has already been fully downloaded.""" - try: - self.to_screen(u'[download] %s has already been downloaded' % file_name) - except (UnicodeEncodeError), err: - self.to_screen(u'[download] The file has already been downloaded') - - def report_unable_to_resume(self): - """Report it was impossible to resume download.""" - self.to_screen(u'[download] Unable to resume') - - def report_finish(self): - """Report download finished.""" - if self.params.get('noprogress', False): - self.to_screen(u'[download] Download completed') - else: - self.to_screen(u'') - - def increment_downloads(self): - """Increment the ordinal that assigns a number to each file.""" - self._num_downloads += 1 - - def prepare_filename(self, info_dict): - """Generate the output filename.""" - try: - template_dict = dict(info_dict) - template_dict['epoch'] = unicode(long(time.time())) - template_dict['autonumber'] = unicode('%05d' % self._num_downloads) - filename = self.params['outtmpl'] % template_dict - return filename - except (ValueError, KeyError), err: - self.trouble(u'ERROR: invalid system charset or erroneous output template') - return None - - def _match_entry(self, info_dict): - """ Returns None iff the file should be downloaded """ - - title = info_dict['title'] - matchtitle = self.params.get('matchtitle', False) - if matchtitle and not re.search(matchtitle, title, re.IGNORECASE): - return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' - rejecttitle = self.params.get('rejecttitle', False) - if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE): - return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' - return None - - def process_info(self, info_dict): - """Process a single dictionary returned by an InfoExtractor.""" - - reason = self._match_entry(info_dict) - if reason is not None: - self.to_screen(u'[download] ' + reason) - return - - max_downloads = self.params.get('max_downloads') - if max_downloads is not None: - if self._num_downloads > int(max_downloads): - raise MaxDownloadsReached() - - filename = self.prepare_filename(info_dict) - - # Forced printings - if self.params.get('forcetitle', False): - print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace') - if self.params.get('forceurl', False): - print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace') - if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict: - print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace') - if self.params.get('forcedescription', False) and 'description' in info_dict: - print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace') - if self.params.get('forcefilename', False) and filename is not None: - print filename.encode(preferredencoding(), 'xmlcharrefreplace') - if self.params.get('forceformat', False): - print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace') - - # Do nothing else if in simulate mode - if self.params.get('simulate', False): - return - - if filename is None: - return - - try: - dn = os.path.dirname(_encodeFilename(filename)) - if dn != '' and not os.path.exists(dn): # dn is already encoded - os.makedirs(dn) - except (OSError, IOError), err: - self.trouble(u'ERROR: unable to create directory ' + unicode(err)) - return - - if self.params.get('writedescription', False): - try: - descfn = filename + u'.description' - self.report_writedescription(descfn) - descfile = open(_encodeFilename(descfn), 'wb') - try: - descfile.write(info_dict['description'].encode('utf-8')) - finally: - descfile.close() - except (OSError, IOError): - self.trouble(u'ERROR: Cannot write description file ' + descfn) - return - - if self.params.get('writeinfojson', False): - infofn = filename + u'.info.json' - self.report_writeinfojson(infofn) - try: - json.dump - except (NameError,AttributeError): - self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.') - return - try: - infof = open(_encodeFilename(infofn), 'wb') - try: - json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',)) - json.dump(json_info_dict, infof) - finally: - infof.close() - except (OSError, IOError): - self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn) - return - - if not self.params.get('skip_download', False): - if self.params.get('nooverwrites', False) and os.path.exists(_encodeFilename(filename)): - success = True - else: - try: - success = self._do_download(filename, info_dict) - except (OSError, IOError), err: - raise UnavailableVideoError - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self.trouble(u'ERROR: unable to download video data: %s' % str(err)) - return - except (ContentTooShortError, ), err: - self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) - return - - if success: - try: - self.post_process(filename, info_dict) - except (PostProcessingError), err: - self.trouble(u'ERROR: postprocessing: %s' % str(err)) - return - - def download(self, url_list): - """Download a given list of URLs.""" - if len(url_list) > 1 and self.fixed_template(): - raise SameFileError(self.params['outtmpl']) - - for url in url_list: - suitable_found = False - for ie in self._ies: - # Go to next InfoExtractor if not suitable - if not ie.suitable(url): - continue - - # Suitable InfoExtractor found - suitable_found = True - - # Extract information from URL and process it - ie.extract(url) - - # Suitable InfoExtractor had been found; go to next URL - break - - if not suitable_found: - self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) - - return self._download_retcode - - def post_process(self, filename, ie_info): - """Run the postprocessing chain on the given file.""" - info = dict(ie_info) - info['filepath'] = filename - for pp in self._pps: - info = pp.run(info) - if info is None: - break - - def _download_with_rtmpdump(self, filename, url, player_url): - self.report_destination(filename) - tmpfilename = self.temp_name(filename) - - # Check for rtmpdump first - try: - subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT) - except (OSError, IOError): - self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run') - return False - - # Download using rtmpdump. rtmpdump returns exit code 2 when - # the connection was interrumpted and resuming appears to be - # possible. This is part of rtmpdump's normal usage, AFAIK. - basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename] - args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)] - if self.params.get('verbose', False): - try: - import pipes - shell_quote = lambda args: ' '.join(map(pipes.quote, args)) - except ImportError: - shell_quote = repr - self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args)) - retval = subprocess.call(args) - while retval == 2 or retval == 1: - prevsize = os.path.getsize(_encodeFilename(tmpfilename)) - self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) - time.sleep(5.0) # This seems to be needed - retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) - cursize = os.path.getsize(_encodeFilename(tmpfilename)) - if prevsize == cursize and retval == 1: - break - # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those - if prevsize == cursize and retval == 2 and cursize > 1024: - self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.') - retval = 0 - break - if retval == 0: - self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(_encodeFilename(tmpfilename))) - self.try_rename(tmpfilename, filename) - return True - else: - self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval) - return False - - def _do_download(self, filename, info_dict): - url = info_dict['url'] - player_url = info_dict.get('player_url', None) - - # Check file already present - if self.params.get('continuedl', False) and os.path.isfile(_encodeFilename(filename)) and not self.params.get('nopart', False): - self.report_file_already_downloaded(filename) - return True - - # Attempt to download using rtmpdump - if url.startswith('rtmp'): - return self._download_with_rtmpdump(filename, url, player_url) - - tmpfilename = self.temp_name(filename) - stream = None - - # Do not include the Accept-Encoding header - headers = {'Youtubedl-no-compression': 'True'} - basic_request = urllib2.Request(url, None, headers) - request = urllib2.Request(url, None, headers) - - # Establish possible resume length - if os.path.isfile(_encodeFilename(tmpfilename)): - resume_len = os.path.getsize(_encodeFilename(tmpfilename)) - else: - resume_len = 0 - - open_mode = 'wb' - if resume_len != 0: - if self.params.get('continuedl', False): - self.report_resuming_byte(resume_len) - request.add_header('Range','bytes=%d-' % resume_len) - open_mode = 'ab' - else: - resume_len = 0 - - count = 0 - retries = self.params.get('retries', 0) - while count <= retries: - # Establish connection - try: - if count == 0 and 'urlhandle' in info_dict: - data = info_dict['urlhandle'] - data = urllib2.urlopen(request) - break - except (urllib2.HTTPError, ), err: - if (err.code < 500 or err.code >= 600) and err.code != 416: - # Unexpected HTTP error - raise - elif err.code == 416: - # Unable to resume (requested range not satisfiable) - try: - # Open the connection again without the range header - data = urllib2.urlopen(basic_request) - content_length = data.info()['Content-Length'] - except (urllib2.HTTPError, ), err: - if err.code < 500 or err.code >= 600: - raise - else: - # Examine the reported length - if (content_length is not None and - (resume_len - 100 < long(content_length) < resume_len + 100)): - # The file had already been fully downloaded. - # Explanation to the above condition: in issue #175 it was revealed that - # YouTube sometimes adds or removes a few bytes from the end of the file, - # changing the file size slightly and causing problems for some users. So - # I decided to implement a suggested change and consider the file - # completely downloaded if the file size differs less than 100 bytes from - # the one in the hard drive. - self.report_file_already_downloaded(filename) - self.try_rename(tmpfilename, filename) - return True - else: - # The length does not match, we start the download over - self.report_unable_to_resume() - open_mode = 'wb' - break - # Retry - count += 1 - if count <= retries: - self.report_retry(count, retries) - - if count > retries: - self.trouble(u'ERROR: giving up after %s retries' % retries) - return False - - data_len = data.info().get('Content-length', None) - if data_len is not None: - data_len = long(data_len) + resume_len - data_len_str = self.format_bytes(data_len) - byte_counter = 0 + resume_len - block_size = 1024 - start = time.time() - while True: - # Download and write - before = time.time() - data_block = data.read(block_size) - after = time.time() - if len(data_block) == 0: - break - byte_counter += len(data_block) - - # Open file just in time - if stream is None: - try: - (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) - assert stream is not None - filename = self.undo_temp_name(tmpfilename) - self.report_destination(filename) - except (OSError, IOError), err: - self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) - return False - try: - stream.write(data_block) - except (IOError, OSError), err: - self.trouble(u'\nERROR: unable to write data: %s' % str(err)) - return False - block_size = self.best_block_size(after - before, len(data_block)) - - # Progress message - speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len) - if data_len is None: - self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA') - else: - percent_str = self.calc_percent(byte_counter, data_len) - eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len) - self.report_progress(percent_str, data_len_str, speed_str, eta_str) - - # Apply rate limit - self.slow_down(start, byte_counter - resume_len) - - if stream is None: - self.trouble(u'\nERROR: Did not get any data blocks') - return False - stream.close() - self.report_finish() - if data_len is not None and byte_counter != data_len: - raise ContentTooShortError(byte_counter, long(data_len)) - self.try_rename(tmpfilename, filename) - - # Update file modification time - if self.params.get('updatetime', True): - info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None)) - - return True - - -class InfoExtractor(object): - """Information Extractor class. - - Information extractors are the classes that, given a URL, extract - information from the video (or videos) the URL refers to. This - information includes the real video URL, the video title and simplified - title, author and others. The information is stored in a dictionary - which is then passed to the FileDownloader. The FileDownloader - processes this information possibly downloading the video to the file - system, among other possible outcomes. The dictionaries must include - the following fields: - - id: Video identifier. - url: Final video URL. - uploader: Nickname of the video uploader. - title: Literal title. - stitle: Simplified title. - ext: Video filename extension. - format: Video format. - player_url: SWF Player URL (may be None). - - The following fields are optional. Their primary purpose is to allow - youtube-dl to serve as the backend for a video search function, such - as the one in youtube2mp3. They are only used when their respective - forced printing functions are called: - - thumbnail: Full URL to a video thumbnail image. - description: One-line video description. - - Subclasses of this one should re-define the _real_initialize() and - _real_extract() methods and define a _VALID_URL regexp. - Probably, they should also be added to the list of extractors. - """ - - _ready = False - _downloader = None - - def __init__(self, downloader=None): - """Constructor. Receives an optional downloader.""" - self._ready = False - self.set_downloader(downloader) - - def suitable(self, url): - """Receives a URL and returns True if suitable for this IE.""" - return re.match(self._VALID_URL, url) is not None - - def initialize(self): - """Initializes an instance (authentication, etc).""" - if not self._ready: - self._real_initialize() - self._ready = True - - def extract(self, url): - """Extracts URL information and returns it in list of dicts.""" - self.initialize() - return self._real_extract(url) - - def set_downloader(self, downloader): - """Sets the downloader for this IE.""" - self._downloader = downloader - - def _real_initialize(self): - """Real initialization process. Redefine in subclasses.""" - pass - - def _real_extract(self, url): - """Real extraction process. Redefine in subclasses.""" - pass - - -class YoutubeIE(InfoExtractor): - """Information extractor for youtube.com.""" - - _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?!view_play_list|my_playlists|artist|playlist)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$' - _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' - _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en' - _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' - _NETRC_MACHINE = 'youtube' - # Listed in order of quality - _available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13'] - _available_formats_prefer_free = ['38', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13'] - _video_extensions = { - '13': '3gp', - '17': 'mp4', - '18': 'mp4', - '22': 'mp4', - '37': 'mp4', - '38': 'video', # You actually don't know if this will be MOV, AVI or whatever - '43': 'webm', - '44': 'webm', - '45': 'webm', - } - _video_dimensions = { - '5': '240x400', - '6': '???', - '13': '???', - '17': '144x176', - '18': '360x640', - '22': '720x1280', - '34': '360x640', - '35': '480x854', - '37': '1080x1920', - '38': '3072x4096', - '43': '360x640', - '44': '480x854', - '45': '720x1280', - } - IE_NAME = u'youtube' - - def report_lang(self): - """Report attempt to set language.""" - self._downloader.to_screen(u'[youtube] Setting language') - - def report_login(self): - """Report attempt to log in.""" - self._downloader.to_screen(u'[youtube] Logging in') - - def report_age_confirmation(self): - """Report attempt to confirm age.""" - self._downloader.to_screen(u'[youtube] Confirming age') - - def report_video_webpage_download(self, video_id): - """Report attempt to download video webpage.""" - self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id) - - def report_video_info_webpage_download(self, video_id): - """Report attempt to download video info webpage.""" - self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id) - - def report_information_extraction(self, video_id): - """Report attempt to extract video information.""" - self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id) - - def report_unavailable_format(self, video_id, format): - """Report extracted video URL.""" - self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format)) - - def report_rtmp_download(self): - """Indicate the download will use the RTMP protocol.""" - self._downloader.to_screen(u'[youtube] RTMP download detected') - - def _print_formats(self, formats): - print 'Available formats:' - for x in formats: - print '%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')) - - def _real_initialize(self): - if self._downloader is None: - return - - username = None - password = None - downloader_params = self._downloader.params - - # Attempt to use provided username and password or .netrc data - if downloader_params.get('username', None) is not None: - username = downloader_params['username'] - password = downloader_params['password'] - elif downloader_params.get('usenetrc', False): - try: - info = netrc.netrc().authenticators(self._NETRC_MACHINE) - if info is not None: - username = info[0] - password = info[2] - else: - raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) - except (IOError, netrc.NetrcParseError), err: - self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err)) - return - - # Set language - request = urllib2.Request(self._LANG_URL) - try: - self.report_lang() - urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err)) - return - - # No authentication to be performed - if username is None: - return - - # Log in - login_form = { - 'current_form': 'loginForm', - 'next': '/', - 'action_login': 'Log In', - 'username': username, - 'password': password, - } - request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form)) - try: - self.report_login() - login_results = urllib2.urlopen(request).read() - if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None: - self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password') - return - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err)) - return - - # Confirm age - age_form = { - 'next_url': '/', - 'action_confirm': 'Confirm', - } - request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form)) - try: - self.report_age_confirmation() - age_results = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err)) - return - - def _real_extract(self, url): - # Extract video id from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - video_id = mobj.group(2) - - # Get video webpage - self.report_video_webpage_download(video_id) - request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id) - try: - video_webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) - return - - # Attempt to extract SWF player URL - mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage) - if mobj is not None: - player_url = re.sub(r'\\(.)', r'\1', mobj.group(1)) - else: - player_url = None - - # Get video info - self.report_video_info_webpage_download(video_id) - for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']: - video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en' - % (video_id, el_type)) - request = urllib2.Request(video_info_url) - try: - video_info_webpage = urllib2.urlopen(request).read() - video_info = parse_qs(video_info_webpage) - if 'token' in video_info: - break - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err)) - return - if 'token' not in video_info: - if 'reason' in video_info: - self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8')) - else: - self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason') - return - - # Start extracting information - self.report_information_extraction(video_id) - - # uploader - if 'author' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract uploader nickname') - return - video_uploader = urllib.unquote_plus(video_info['author'][0]) - - # title - if 'title' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = urllib.unquote_plus(video_info['title'][0]) - video_title = video_title.decode('utf-8') - video_title = sanitize_title(video_title) - - # simplified title - simple_title = _simplify_title(video_title) - - # thumbnail image - if 'thumbnail_url' not in video_info: - self._downloader.trouble(u'WARNING: unable to extract video thumbnail') - video_thumbnail = '' - else: # don't panic if we can't find it - video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0]) - - # upload date - upload_date = u'NA' - mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL) - if mobj is not None: - upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split()) - format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y'] - for expression in format_expressions: - try: - upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d') - except: - pass - - # description - try: - lxml.etree - except NameError: - video_description = u'No description available.' - mobj = re.search(r'<meta name="description" content="(.*?)">', video_webpage) - if mobj is not None: - video_description = mobj.group(1).decode('utf-8') - else: - html_parser = lxml.etree.HTMLParser(encoding='utf-8') - vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser) - video_description = u''.join(vwebpage_doc.xpath('id("eow-description")//text()')) - # TODO use another parser - - # token - video_token = urllib.unquote_plus(video_info['token'][0]) - - # Decide which formats to download - req_format = self._downloader.params.get('format', None) - - if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'): - self.report_rtmp_download() - video_url_list = [(None, video_info['conn'][0])] - elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1: - url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',') - url_data = [parse_qs(uds) for uds in url_data_strs] - url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data) - url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data) - - format_limit = self._downloader.params.get('format_limit', None) - available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats - if format_limit is not None and format_limit in available_formats: - format_list = available_formats[available_formats.index(format_limit):] - else: - format_list = available_formats - existing_formats = [x for x in format_list if x in url_map] - if len(existing_formats) == 0: - self._downloader.trouble(u'ERROR: no known formats available for video') - return - if self._downloader.params.get('listformats', None): - self._print_formats(existing_formats) - return - if req_format is None or req_format == 'best': - video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality - elif req_format == 'worst': - video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality - elif req_format in ('-1', 'all'): - video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats - else: - # Specific formats. We pick the first in a slash-delimeted sequence. - # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'. - req_formats = req_format.split('/') - video_url_list = None - for rf in req_formats: - if rf in url_map: - video_url_list = [(rf, url_map[rf])] - break - if video_url_list is None: - self._downloader.trouble(u'ERROR: requested format not available') - return - else: - self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info') - return - - for format_param, video_real_url in video_url_list: - # At this point we have a new video - self._downloader.increment_downloads() - - # Extension - video_extension = self._video_extensions.get(format_param, 'flv') - - try: - # Process video information - self._downloader.process_info({ - 'id': video_id.decode('utf-8'), - 'url': video_real_url.decode('utf-8'), - 'uploader': video_uploader.decode('utf-8'), - 'upload_date': upload_date, - 'title': video_title, - 'stitle': simple_title, - 'ext': video_extension.decode('utf-8'), - 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), - 'thumbnail': video_thumbnail.decode('utf-8'), - 'description': video_description, - 'player_url': player_url, - }) - except UnavailableVideoError, err: - self._downloader.trouble(u'\nERROR: unable to download video') - - -class MetacafeIE(InfoExtractor): - """Information Extractor for metacafe.com.""" - - _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*' - _DISCLAIMER = 'http://www.metacafe.com/family_filter/' - _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user' - _youtube_ie = None - IE_NAME = u'metacafe' - - def __init__(self, youtube_ie, downloader=None): - InfoExtractor.__init__(self, downloader) - self._youtube_ie = youtube_ie - - def report_disclaimer(self): - """Report disclaimer retrieval.""" - self._downloader.to_screen(u'[metacafe] Retrieving disclaimer') - - def report_age_confirmation(self): - """Report attempt to confirm age.""" - self._downloader.to_screen(u'[metacafe] Confirming age') - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id) - - def _real_initialize(self): - # Retrieve disclaimer - request = urllib2.Request(self._DISCLAIMER) - try: - self.report_disclaimer() - disclaimer = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err)) - return - - # Confirm age - disclaimer_form = { - 'filters': '0', - 'submit': "Continue - I'm over 18", - } - request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form)) - try: - self.report_age_confirmation() - disclaimer = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err)) - return - - def _real_extract(self, url): - # Extract id and simplified title from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - - video_id = mobj.group(1) - - # Check if video comes from YouTube - mobj2 = re.match(r'^yt-(.*)$', video_id) - if mobj2 is not None: - self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1)) - return - - # At this point we have a new video - self._downloader.increment_downloads() - - simple_title = mobj.group(2).decode('utf-8') - - # Retrieve video webpage to extract further information - request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id) - try: - self.report_download_webpage(video_id) - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err)) - return - - # Extract URL, uploader and title from webpage - self.report_extraction(video_id) - mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage) - if mobj is not None: - mediaURL = urllib.unquote(mobj.group(1)) - video_extension = mediaURL[-3:] - - # Extract gdaKey if available - mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage) - if mobj is None: - video_url = mediaURL - else: - gdaKey = mobj.group(1) - video_url = '%s?__gda__=%s' % (mediaURL, gdaKey) - else: - mobj = re.search(r' name="flashvars" value="(.*?)"', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') - return - vardict = parse_qs(mobj.group(1)) - if 'mediaData' not in vardict: - self._downloader.trouble(u'ERROR: unable to extract media URL') - return - mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0]) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') - return - mediaURL = mobj.group(1).replace('\\/', '/') - video_extension = mediaURL[-3:] - video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2)) - - mobj = re.search(r'(?im)<title>(.*) - Video', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') - return - video_title = mobj.group(1).decode('utf-8') - video_title = sanitize_title(video_title) - - mobj = re.search(r'(?ms)By:\s*(.+?)<', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract uploader nickname') - return - video_uploader = mobj.group(1) - - try: - # Process video information - self._downloader.process_info({ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': video_uploader.decode('utf-8'), - 'upload_date': u'NA', - 'title': video_title, - 'stitle': simple_title, - 'ext': video_extension.decode('utf-8'), - 'format': u'NA', - 'player_url': None, - }) - except UnavailableVideoError: - self._downloader.trouble(u'\nERROR: unable to download video') - - -class DailymotionIE(InfoExtractor): - """Information Extractor for Dailymotion""" - - _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)' - IE_NAME = u'dailymotion' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id) - - def _real_extract(self, url): - # Extract id and simplified title from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - - # At this point we have a new video - self._downloader.increment_downloads() - video_id = mobj.group(1) - - video_extension = 'flv' - - # Retrieve video webpage to extract further information - request = urllib2.Request(url) - request.add_header('Cookie', 'family_filter=off') - try: - self.report_download_webpage(video_id) - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err)) - return - - # Extract URL, uploader and title from webpage - self.report_extraction(video_id) - mobj = re.search(r'(?i)addVariable\(\"sequence\"\s*,\s*\"([^\"]+?)\"\)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') - return - sequence = urllib.unquote(mobj.group(1)) - mobj = re.search(r',\"sdURL\"\:\"([^\"]+?)\",', sequence) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') - return - mediaURL = urllib.unquote(mobj.group(1)).replace('\\', '') - - # if needed add http://www.dailymotion.com/ if relative URL - - video_url = mediaURL - - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') - return - video_title = _unescapeHTML(mobj.group('title').decode('utf-8')) - video_title = sanitize_title(video_title) - simple_title = _simplify_title(video_title) - - mobj = re.search(r'(?im)[^<]+?]+?>([^<]+?)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract uploader nickname') - return - video_uploader = mobj.group(1) - - try: - # Process video information - self._downloader.process_info({ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': video_uploader.decode('utf-8'), - 'upload_date': u'NA', - 'title': video_title, - 'stitle': simple_title, - 'ext': video_extension.decode('utf-8'), - 'format': u'NA', - 'player_url': None, - }) - except UnavailableVideoError: - self._downloader.trouble(u'\nERROR: unable to download video') - - -class GoogleIE(InfoExtractor): - """Information extractor for video.google.com.""" - - _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*' - IE_NAME = u'video.google' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id) - - def _real_extract(self, url): - # Extract id from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) - return - - # At this point we have a new video - self._downloader.increment_downloads() - video_id = mobj.group(1) - - video_extension = 'mp4' - - # Retrieve video webpage to extract further information - request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id) - try: - self.report_download_webpage(video_id) - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) - return - - # Extract URL, uploader, and title from webpage - self.report_extraction(video_id) - mobj = re.search(r"download_url:'([^']+)'", webpage) - if mobj is None: - video_extension = 'flv' - mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') - return - mediaURL = urllib.unquote(mobj.group(1)) - mediaURL = mediaURL.replace('\\x3d', '\x3d') - mediaURL = mediaURL.replace('\\x26', '\x26') - - video_url = mediaURL - - mobj = re.search(r'(.*)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') - return - video_title = mobj.group(1).decode('utf-8') - video_title = sanitize_title(video_title) - simple_title = _simplify_title(video_title) - - # Extract video description - mobj = re.search(r'([^<]*)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video description') - return - video_description = mobj.group(1).decode('utf-8') - if not video_description: - video_description = 'No description available.' - - # Extract video thumbnail - if self._downloader.params.get('forcethumbnail', False): - request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id))) - try: - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) - return - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video thumbnail') - return - video_thumbnail = mobj.group(1) - else: # we need something to pass to process_info - video_thumbnail = '' - - try: - # Process video information - self._downloader.process_info({ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': u'NA', - 'upload_date': u'NA', - 'title': video_title, - 'stitle': simple_title, - 'ext': video_extension.decode('utf-8'), - 'format': u'NA', - 'player_url': None, - }) - except UnavailableVideoError: - self._downloader.trouble(u'\nERROR: unable to download video') - - -class PhotobucketIE(InfoExtractor): - """Information extractor for photobucket.com.""" - - _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)' - IE_NAME = u'photobucket' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id) - - def _real_extract(self, url): - # Extract id from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) - return - - # At this point we have a new video - self._downloader.increment_downloads() - video_id = mobj.group(1) - - video_extension = 'flv' - - # Retrieve video webpage to extract further information - request = urllib2.Request(url) - try: - self.report_download_webpage(video_id) - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) - return - - # Extract URL, uploader, and title from webpage - self.report_extraction(video_id) - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') - return - mediaURL = urllib.unquote(mobj.group(1)) - - video_url = mediaURL - - mobj = re.search(r'(.*) video by (.*) - Photobucket', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') - return - video_title = mobj.group(1).decode('utf-8') - video_title = sanitize_title(video_title) - simple_title = _simplify_title(vide_title) - - video_uploader = mobj.group(2).decode('utf-8') - - try: - # Process video information - self._downloader.process_info({ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': video_uploader, - 'upload_date': u'NA', - 'title': video_title, - 'stitle': simple_title, - 'ext': video_extension.decode('utf-8'), - 'format': u'NA', - 'player_url': None, - }) - except UnavailableVideoError: - self._downloader.trouble(u'\nERROR: unable to download video') - - -class YahooIE(InfoExtractor): - """Information extractor for video.yahoo.com.""" - - # _VALID_URL matches all Yahoo! Video URLs - # _VPAGE_URL matches only the extractable '/watch/' URLs - _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?' - _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?' - IE_NAME = u'video.yahoo' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id) - - def _real_extract(self, url, new_video=True): - # Extract ID from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) - return - - # At this point we have a new video - self._downloader.increment_downloads() - video_id = mobj.group(2) - video_extension = 'flv' - - # Rewrite valid but non-extractable URLs as - # extractable English language /watch/ URLs - if re.match(self._VPAGE_URL, url) is None: - request = urllib2.Request(url) - try: - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) - return - - mobj = re.search(r'\("id", "([0-9]+)"\);', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: Unable to extract id field') - return - yahoo_id = mobj.group(1) - - mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: Unable to extract vid field') - return - yahoo_vid = mobj.group(1) - - url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id) - return self._real_extract(url, new_video=False) - - # Retrieve video webpage to extract further information - request = urllib2.Request(url) - try: - self.report_download_webpage(video_id) - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) - return - - # Extract uploader and title from webpage - self.report_extraction(video_id) - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = mobj.group(1).decode('utf-8') - simple_title = _simplify_title(video_title) - - mobj = re.search(r'

(.*)

', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video uploader') - return - video_uploader = mobj.group(1).decode('utf-8') - - # Extract video thumbnail - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video thumbnail') - return - video_thumbnail = mobj.group(1).decode('utf-8') - - # Extract video description - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video description') - return - video_description = mobj.group(1).decode('utf-8') - if not video_description: - video_description = 'No description available.' - - # Extract video height and width - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video height') - return - yv_video_height = mobj.group(1) - - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video width') - return - yv_video_width = mobj.group(1) - - # Retrieve video playlist to extract media URL - # I'm not completely sure what all these options are, but we - # seem to need most of them, otherwise the server sends a 401. - yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents - yv_bitrate = '700' # according to Wikipedia this is hard-coded - request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id + - '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height + - '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797') - try: - self.report_download_webpage(video_id) - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) - return - - # Extract media URL from playlist XML - mobj = re.search(r'(.*?)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = mobj.group(1).decode('utf-8') - simple_title = _simplify_title(video_title) - - # Extract uploader - mobj = re.search(r'http://vimeo.com/(.*?)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video uploader') - return - video_uploader = mobj.group(1).decode('utf-8') - - # Extract video thumbnail - mobj = re.search(r'(.*?)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video thumbnail') - return - video_thumbnail = mobj.group(1).decode('utf-8') - - # # Extract video description - # mobj = re.search(r'', webpage) - # if mobj is None: - # self._downloader.trouble(u'ERROR: unable to extract video description') - # return - # video_description = mobj.group(1).decode('utf-8') - # if not video_description: video_description = 'No description available.' - video_description = 'Foo.' - - # Vimeo specific: extract request signature - mobj = re.search(r'(.*?)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract request signature') - return - sig = mobj.group(1).decode('utf-8') - - # Vimeo specific: extract video quality information - mobj = re.search(r'(\d+)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video quality information') - return - quality = mobj.group(1).decode('utf-8') - - if int(quality) == 1: - quality = 'hd' - else: - quality = 'sd' - - # Vimeo specific: Extract request signature expiration - mobj = re.search(r'(.*?)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract request signature expiration') - return - sig_exp = mobj.group(1).decode('utf-8') - - video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s/?q=%s" % (video_id, sig, sig_exp, quality) - - try: - # Process video information - self._downloader.process_info({ - 'id': video_id.decode('utf-8'), - 'url': video_url, - 'uploader': video_uploader, - 'upload_date': u'NA', - 'title': video_title, - 'stitle': simple_title, - 'ext': u'mp4', - 'thumbnail': video_thumbnail.decode('utf-8'), - 'description': video_description, - 'thumbnail': video_thumbnail, - 'description': video_description, - 'player_url': None, - }) - except UnavailableVideoError: - self._downloader.trouble(u'ERROR: unable to download video') - - -class GenericIE(InfoExtractor): - """Generic last-resort information extractor.""" - - _VALID_URL = r'.*' - IE_NAME = u'generic' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.') - self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id) - - def _real_extract(self, url): - # At this point we have a new video - self._downloader.increment_downloads() - - video_id = url.split('/')[-1] - request = urllib2.Request(url) - try: - self.report_download_webpage(video_id) - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) - return - except ValueError, err: - # since this is the last-resort InfoExtractor, if - # this error is thrown, it'll be thrown here - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) - return - - self.report_extraction(video_id) - # Start with something easy: JW Player in SWFObject - mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) - if mobj is None: - # Broaden the search a little bit - mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) - return - - # It's possible that one of the regexes - # matched, but returned an empty group: - if mobj.group(1) is None: - self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) - return - - video_url = urllib.unquote(mobj.group(1)) - video_id = os.path.basename(video_url) - - # here's a fun little line of code for you: - video_extension = os.path.splitext(video_id)[1][1:] - video_id = os.path.splitext(video_id)[0] - - # it's tempting to parse this further, but you would - # have to take into account all the variations like - # Video Title - Site Name - # Site Name | Video Title - # Video Title - Tagline | Site Name - # and so on and so forth; it's just not practical - mobj = re.search(r'(.*)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') - return - video_title = mobj.group(1).decode('utf-8') - video_title = sanitize_title(video_title) - simple_title = _simplify_title(video_title) - - # video uploader is domain name - mobj = re.match(r'(?:https?://)?([^/]*)/.*', url) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') - return - video_uploader = mobj.group(1).decode('utf-8') - - try: - # Process video information - self._downloader.process_info({ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': video_uploader, - 'upload_date': u'NA', - 'title': video_title, - 'stitle': simple_title, - 'ext': video_extension.decode('utf-8'), - 'format': u'NA', - 'player_url': None, - }) - except UnavailableVideoError, err: - self._downloader.trouble(u'\nERROR: unable to download video') - - -class YoutubeSearchIE(InfoExtractor): - """Information Extractor for YouTube search queries.""" - _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+' - _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en' - _VIDEO_INDICATOR = r'href="/watch\?v=.+?"' - _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*' - _youtube_ie = None - _max_youtube_results = 1000 - IE_NAME = u'youtube:search' - - def __init__(self, youtube_ie, downloader=None): - InfoExtractor.__init__(self, downloader) - self._youtube_ie = youtube_ie - - def report_download_page(self, query, pagenum): - """Report attempt to download playlist page with given number.""" - query = query.decode(preferredencoding()) - self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum)) - - def _real_initialize(self): - self._youtube_ie.initialize() - - def _real_extract(self, query): - mobj = re.match(self._VALID_URL, query) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) - return - - prefix, query = query.split(':') - prefix = prefix[8:] - query = query.encode('utf-8') - if prefix == '': - self._download_n_results(query, 1) - return - elif prefix == 'all': - self._download_n_results(query, self._max_youtube_results) - return - else: - try: - n = long(prefix) - if n <= 0: - self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) - return - elif n > self._max_youtube_results: - self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n)) - n = self._max_youtube_results - self._download_n_results(query, n) - return - except ValueError: # parsing prefix as integer fails - self._download_n_results(query, 1) - return - - def _download_n_results(self, query, n): - """Downloads a specified number of results for a query""" - - video_ids = [] - already_seen = set() - pagenum = 1 - - while True: - self.report_download_page(query, pagenum) - result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) - request = urllib2.Request(result_url) - try: - page = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) - return - - # Extract video identifiers - for mobj in re.finditer(self._VIDEO_INDICATOR, page): - video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1] - if video_id not in already_seen: - video_ids.append(video_id) - already_seen.add(video_id) - if len(video_ids) == n: - # Specified n videos reached - for id in video_ids: - self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id) - return - - if re.search(self._MORE_PAGES_INDICATOR, page) is None: - for id in video_ids: - self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id) - return - - pagenum = pagenum + 1 - - -class GoogleSearchIE(InfoExtractor): - """Information Extractor for Google Video search queries.""" - _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+' - _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en' - _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&' - _MORE_PAGES_INDICATOR = r'Next' - _google_ie = None - _max_google_results = 1000 - IE_NAME = u'video.google:search' - - def __init__(self, google_ie, downloader=None): - InfoExtractor.__init__(self, downloader) - self._google_ie = google_ie - - def report_download_page(self, query, pagenum): - """Report attempt to download playlist page with given number.""" - query = query.decode(preferredencoding()) - self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum)) - - def _real_initialize(self): - self._google_ie.initialize() - - def _real_extract(self, query): - mobj = re.match(self._VALID_URL, query) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) - return - - prefix, query = query.split(':') - prefix = prefix[8:] - query = query.encode('utf-8') - if prefix == '': - self._download_n_results(query, 1) - return - elif prefix == 'all': - self._download_n_results(query, self._max_google_results) - return - else: - try: - n = long(prefix) - if n <= 0: - self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) - return - elif n > self._max_google_results: - self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n)) - n = self._max_google_results - self._download_n_results(query, n) - return - except ValueError: # parsing prefix as integer fails - self._download_n_results(query, 1) - return - - def _download_n_results(self, query, n): - """Downloads a specified number of results for a query""" - - video_ids = [] - already_seen = set() - pagenum = 1 - - while True: - self.report_download_page(query, pagenum) - result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) - request = urllib2.Request(result_url) - try: - page = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) - return - - # Extract video identifiers - for mobj in re.finditer(self._VIDEO_INDICATOR, page): - video_id = mobj.group(1) - if video_id not in already_seen: - video_ids.append(video_id) - already_seen.add(video_id) - if len(video_ids) == n: - # Specified n videos reached - for id in video_ids: - self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id) - return - - if re.search(self._MORE_PAGES_INDICATOR, page) is None: - for id in video_ids: - self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id) - return - - pagenum = pagenum + 1 - - -class YahooSearchIE(InfoExtractor): - """Information Extractor for Yahoo! Video search queries.""" - _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+' - _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s' - _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"' - _MORE_PAGES_INDICATOR = r'\s*Next' - _yahoo_ie = None - _max_yahoo_results = 1000 - IE_NAME = u'video.yahoo:search' - - def __init__(self, yahoo_ie, downloader=None): - InfoExtractor.__init__(self, downloader) - self._yahoo_ie = yahoo_ie - - def report_download_page(self, query, pagenum): - """Report attempt to download playlist page with given number.""" - query = query.decode(preferredencoding()) - self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum)) - - def _real_initialize(self): - self._yahoo_ie.initialize() - - def _real_extract(self, query): - mobj = re.match(self._VALID_URL, query) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) - return - - prefix, query = query.split(':') - prefix = prefix[8:] - query = query.encode('utf-8') - if prefix == '': - self._download_n_results(query, 1) - return - elif prefix == 'all': - self._download_n_results(query, self._max_yahoo_results) - return - else: - try: - n = long(prefix) - if n <= 0: - self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) - return - elif n > self._max_yahoo_results: - self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n)) - n = self._max_yahoo_results - self._download_n_results(query, n) - return - except ValueError: # parsing prefix as integer fails - self._download_n_results(query, 1) - return - - def _download_n_results(self, query, n): - """Downloads a specified number of results for a query""" - - video_ids = [] - already_seen = set() - pagenum = 1 - - while True: - self.report_download_page(query, pagenum) - result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) - request = urllib2.Request(result_url) - try: - page = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) - return - - # Extract video identifiers - for mobj in re.finditer(self._VIDEO_INDICATOR, page): - video_id = mobj.group(1) - if video_id not in already_seen: - video_ids.append(video_id) - already_seen.add(video_id) - if len(video_ids) == n: - # Specified n videos reached - for id in video_ids: - self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id) - return - - if re.search(self._MORE_PAGES_INDICATOR, page) is None: - for id in video_ids: - self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id) - return - - pagenum = pagenum + 1 - - -class YoutubePlaylistIE(InfoExtractor): - """Information Extractor for YouTube playlists.""" - - _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*' - _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en' - _VIDEO_INDICATOR = r'/watch\?v=(.+?)&' - _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*' - _youtube_ie = None - IE_NAME = u'youtube:playlist' - - def __init__(self, youtube_ie, downloader=None): - InfoExtractor.__init__(self, downloader) - self._youtube_ie = youtube_ie - - def report_download_page(self, playlist_id, pagenum): - """Report attempt to download playlist page with given number.""" - self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum)) - - def _real_initialize(self): - self._youtube_ie.initialize() - - def _real_extract(self, url): - # Extract playlist id - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid url: %s' % url) - return - - # Single video case - if mobj.group(3) is not None: - self._youtube_ie.extract(mobj.group(3)) - return - - # Download playlist pages - # prefix is 'p' as default for playlists but there are other types that need extra care - playlist_prefix = mobj.group(1) - if playlist_prefix == 'a': - playlist_access = 'artist' - else: - playlist_prefix = 'p' - playlist_access = 'view_play_list' - playlist_id = mobj.group(2) - video_ids = [] - pagenum = 1 - - while True: - self.report_download_page(playlist_id, pagenum) - url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum) - request = urllib2.Request(url) - try: - page = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) - return - - # Extract video identifiers - ids_in_page = [] - for mobj in re.finditer(self._VIDEO_INDICATOR, page): - if mobj.group(1) not in ids_in_page: - ids_in_page.append(mobj.group(1)) - video_ids.extend(ids_in_page) - - if re.search(self._MORE_PAGES_INDICATOR, page) is None: - break - pagenum = pagenum + 1 - - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - video_ids = video_ids[playliststart:playlistend] - - for id in video_ids: - self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id) - return - - -class YoutubeUserIE(InfoExtractor): - """Information Extractor for YouTube users.""" - - _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)' - _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s' - _GDATA_PAGE_SIZE = 50 - _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d' - _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]' - _youtube_ie = None - IE_NAME = u'youtube:user' - - def __init__(self, youtube_ie, downloader=None): - InfoExtractor.__init__(self, downloader) - self._youtube_ie = youtube_ie - - def report_download_page(self, username, start_index): - """Report attempt to download user page.""" - self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' % - (username, start_index, start_index + self._GDATA_PAGE_SIZE)) - - def _real_initialize(self): - self._youtube_ie.initialize() - - def _real_extract(self, url): - # Extract username - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid url: %s' % url) - return - - username = mobj.group(1) - - # Download video ids using YouTube Data API. Result size per - # query is limited (currently to 50 videos) so we need to query - # page by page until there are no video ids - it means we got - # all of them. - - video_ids = [] - pagenum = 0 - - while True: - start_index = pagenum * self._GDATA_PAGE_SIZE + 1 - self.report_download_page(username, start_index) - - request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)) - - try: - page = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err)) - return - - # Extract video identifiers - ids_in_page = [] - - for mobj in re.finditer(self._VIDEO_INDICATOR, page): - if mobj.group(1) not in ids_in_page: - ids_in_page.append(mobj.group(1)) - - video_ids.extend(ids_in_page) - - # A little optimization - if current page is not - # "full", ie. does not contain PAGE_SIZE video ids then - # we can assume that this page is the last one - there - # are no more ids on further pages - no need to query - # again. - - if len(ids_in_page) < self._GDATA_PAGE_SIZE: - break - - pagenum += 1 - - all_ids_count = len(video_ids) - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - - if playlistend == -1: - video_ids = video_ids[playliststart:] - else: - video_ids = video_ids[playliststart:playlistend] - - self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" % - (username, all_ids_count, len(video_ids))) - - for video_id in video_ids: - self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id) - - -class DepositFilesIE(InfoExtractor): - """Information extractor for depositfiles.com""" - - _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)' - IE_NAME = u'DepositFiles' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_download_webpage(self, file_id): - """Report webpage download.""" - self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id) - - def report_extraction(self, file_id): - """Report information extraction.""" - self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id) - - def _real_extract(self, url): - # At this point we have a new file - self._downloader.increment_downloads() - - file_id = url.split('/')[-1] - # Rebuild url in english locale - url = 'http://depositfiles.com/en/files/' + file_id - - # Retrieve file webpage with 'Free download' button pressed - free_download_indication = { 'gateway_result' : '1' } - request = urllib2.Request(url, urllib.urlencode(free_download_indication)) - try: - self.report_download_webpage(file_id) - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err)) - return - - # Search for the real file URL - mobj = re.search(r'(Attention.*?)', webpage, re.DOTALL) - if (mobj is not None) and (mobj.group(1) is not None): - restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip() - self._downloader.trouble(u'ERROR: %s' % restriction_message) - else: - self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url) - return - - file_url = mobj.group(1) - file_extension = os.path.splitext(file_url)[1][1:] - - # Search for file title - mobj = re.search(r'', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') - return - file_title = mobj.group(1).decode('utf-8') - - try: - # Process file information - self._downloader.process_info({ - 'id': file_id.decode('utf-8'), - 'url': file_url.decode('utf-8'), - 'uploader': u'NA', - 'upload_date': u'NA', - 'title': file_title, - 'stitle': file_title, - 'ext': file_extension.decode('utf-8'), - 'format': u'NA', - 'player_url': None, - }) - except UnavailableVideoError, err: - self._downloader.trouble(u'ERROR: unable to download file') - - -class FacebookIE(InfoExtractor): - """Information Extractor for Facebook""" - - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P\d+)(?:.*)' - _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&' - _NETRC_MACHINE = 'facebook' - _available_formats = ['video', 'highqual', 'lowqual'] - _video_extensions = { - 'video': 'mp4', - 'highqual': 'mp4', - 'lowqual': 'mp4', - } - IE_NAME = u'facebook' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def _reporter(self, message): - """Add header and report message.""" - self._downloader.to_screen(u'[facebook] %s' % message) - - def report_login(self): - """Report attempt to log in.""" - self._reporter(u'Logging in') - - def report_video_webpage_download(self, video_id): - """Report attempt to download video webpage.""" - self._reporter(u'%s: Downloading video webpage' % video_id) - - def report_information_extraction(self, video_id): - """Report attempt to extract video information.""" - self._reporter(u'%s: Extracting video information' % video_id) - - def _parse_page(self, video_webpage): - """Extract video information from page""" - # General data - data = {'title': r'\("video_title", "(.*?)"\)', - 'description': r'
(.*?)
', - 'owner': r'\("video_owner_name", "(.*?)"\)', - 'thumbnail': r'\("thumb_url", "(?P.*?)"\)', - } - video_info = {} - for piece in data.keys(): - mobj = re.search(data[piece], video_webpage) - if mobj is not None: - video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape")) - - # Video urls - video_urls = {} - for fmt in self._available_formats: - mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage) - if mobj is not None: - # URL is in a Javascript segment inside an escaped Unicode format within - # the generally utf-8 page - video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape")) - video_info['video_urls'] = video_urls - - return video_info - - def _real_initialize(self): - if self._downloader is None: - return - - useremail = None - password = None - downloader_params = self._downloader.params - - # Attempt to use provided username and password or .netrc data - if downloader_params.get('username', None) is not None: - useremail = downloader_params['username'] - password = downloader_params['password'] - elif downloader_params.get('usenetrc', False): - try: - info = netrc.netrc().authenticators(self._NETRC_MACHINE) - if info is not None: - useremail = info[0] - password = info[2] - else: - raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) - except (IOError, netrc.NetrcParseError), err: - self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err)) - return - - if useremail is None: - return - - # Log in - login_form = { - 'email': useremail, - 'pass': password, - 'login': 'Log+In' - } - request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form)) - try: - self.report_login() - login_results = urllib2.urlopen(request).read() - if re.search(r'', login_results) is not None: - self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.') - return - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err)) - return - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - video_id = mobj.group('ID') - - # Get video webpage - self.report_video_webpage_download(video_id) - request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id) - try: - page = urllib2.urlopen(request) - video_webpage = page.read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) - return - - # Start extracting information - self.report_information_extraction(video_id) - - # Extract information - video_info = self._parse_page(video_webpage) - - # uploader - if 'owner' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract uploader nickname') - return - video_uploader = video_info['owner'] - - # title - if 'title' not in video_info: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = video_info['title'] - video_title = video_title.decode('utf-8') - video_title = sanitize_title(video_title) - - simple_title = _simplify_title(video_title) - - # thumbnail image - if 'thumbnail' not in video_info: - self._downloader.trouble(u'WARNING: unable to extract video thumbnail') - video_thumbnail = '' - else: - video_thumbnail = video_info['thumbnail'] - - # upload date - upload_date = u'NA' - if 'upload_date' in video_info: - upload_time = video_info['upload_date'] - timetuple = email.utils.parsedate_tz(upload_time) - if timetuple is not None: - try: - upload_date = time.strftime('%Y%m%d', timetuple[0:9]) - except: - pass - - # description - video_description = video_info.get('description', 'No description available.') - - url_map = video_info['video_urls'] - if len(url_map.keys()) > 0: - # Decide which formats to download - req_format = self._downloader.params.get('format', None) - format_limit = self._downloader.params.get('format_limit', None) - - if format_limit is not None and format_limit in self._available_formats: - format_list = self._available_formats[self._available_formats.index(format_limit):] - else: - format_list = self._available_formats - existing_formats = [x for x in format_list if x in url_map] - if len(existing_formats) == 0: - self._downloader.trouble(u'ERROR: no known formats available for video') - return - if req_format is None: - video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality - elif req_format == 'worst': - video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality - elif req_format == '-1': - video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats - else: - # Specific format - if req_format not in url_map: - self._downloader.trouble(u'ERROR: requested format not available') - return - video_url_list = [(req_format, url_map[req_format])] # Specific format - - for format_param, video_real_url in video_url_list: - - # At this point we have a new video - self._downloader.increment_downloads() - - # Extension - video_extension = self._video_extensions.get(format_param, 'mp4') - - try: - # Process video information - self._downloader.process_info({ - 'id': video_id.decode('utf-8'), - 'url': video_real_url.decode('utf-8'), - 'uploader': video_uploader.decode('utf-8'), - 'upload_date': upload_date, - 'title': video_title, - 'stitle': simple_title, - 'ext': video_extension.decode('utf-8'), - 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), - 'thumbnail': video_thumbnail.decode('utf-8'), - 'description': video_description.decode('utf-8'), - 'player_url': None, - }) - except UnavailableVideoError, err: - self._downloader.trouble(u'\nERROR: unable to download video') - -class BlipTVIE(InfoExtractor): - """Information extractor for blip.tv""" - - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$' - _URL_EXT = r'^.*\.([a-z0-9]+)$' - IE_NAME = u'blip.tv' - - def report_extraction(self, file_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id)) - - def report_direct_download(self, title): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title)) - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - - if '?' in url: - cchar = '&' - else: - cchar = '?' - json_url = url + cchar + 'skin=json&version=2&no_wrap=1' - request = urllib2.Request(json_url) - self.report_extraction(mobj.group(1)) - info = None - try: - urlh = urllib2.urlopen(request) - if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download - basename = url.split('/')[-1] - title,ext = os.path.splitext(basename) - title = title.decode('UTF-8') - ext = ext.replace('.', '') - self.report_direct_download(title) - info = { - 'id': title, - 'url': url, - 'title': title, - 'stitle': _simplify_title(title), - 'ext': ext, - 'urlhandle': urlh - } - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err)) - return - if info is None: # Regular URL - try: - json_code = urlh.read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % str(err)) - return - - try: - json_data = json.loads(json_code) - if 'Post' in json_data: - data = json_data['Post'] - else: - data = json_data - - upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d') - video_url = data['media']['url'] - umobj = re.match(self._URL_EXT, video_url) - if umobj is None: - raise ValueError('Can not determine filename extension') - ext = umobj.group(1) - - info = { - 'id': data['item_id'], - 'url': video_url, - 'uploader': data['display_name'], - 'upload_date': upload_date, - 'title': data['title'], - 'stitle': _simplify_title(data['title']), - 'ext': ext, - 'format': data['media']['mimeType'], - 'thumbnail': data['thumbnailUrl'], - 'description': data['description'], - 'player_url': data['embedUrl'] - } - except (ValueError,KeyError), err: - self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err)) - return - - self._downloader.increment_downloads() - - try: - self._downloader.process_info(info) - except UnavailableVideoError, err: - self._downloader.trouble(u'\nERROR: unable to download video') - - -class MyVideoIE(InfoExtractor): - """Information Extractor for myvideo.de.""" - - _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*' - IE_NAME = u'myvideo' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id) - - def _real_extract(self,url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._download.trouble(u'ERROR: invalid URL: %s' % url) - return - - video_id = mobj.group(1) - - # Get video webpage - request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id) - try: - self.report_download_webpage(video_id) - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err)) - return - - self.report_extraction(video_id) - mobj = re.search(r'', - webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract media URL') - return - video_url = mobj.group(1) + ('/%s.flv' % video_id) - - mobj = re.search('([^<]+)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract title') - return - - video_title = mobj.group(1) - video_title = sanitize_title(video_title) - - simple_title = _simplify_title(video_title) - - try: - self._downloader.process_info({ - 'id': video_id, - 'url': video_url, - 'uploader': u'NA', - 'upload_date': u'NA', - 'title': video_title, - 'stitle': simple_title, - 'ext': u'flv', - 'format': u'NA', - 'player_url': None, - }) - except UnavailableVideoError: - self._downloader.trouble(u'\nERROR: Unable to download video') - -class ComedyCentralIE(InfoExtractor): - """Information extractor for The Daily Show and Colbert Report """ - - _VALID_URL = r'^(:(?Ptds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?Pthedailyshow|colbertnation)\.com/full-episodes/(?P.*)$' - IE_NAME = u'comedycentral' - - def report_extraction(self, episode_id): - self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id) - - def report_config_download(self, episode_id): - self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id) - - def report_index_download(self, episode_id): - self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id) - - def report_player_url(self, episode_id): - self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id) - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - - if mobj.group('shortname'): - if mobj.group('shortname') in ('tds', 'thedailyshow'): - url = u'http://www.thedailyshow.com/full-episodes/' - else: - url = u'http://www.colbertnation.com/full-episodes/' - mobj = re.match(self._VALID_URL, url) - assert mobj is not None - - dlNewest = not mobj.group('episode') - if dlNewest: - epTitle = mobj.group('showname') - else: - epTitle = mobj.group('episode') - - req = urllib2.Request(url) - self.report_extraction(epTitle) - try: - htmlHandle = urllib2.urlopen(req) - html = htmlHandle.read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err)) - return - if dlNewest: - url = htmlHandle.geturl() - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url) - return - if mobj.group('episode') == '': - self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url) - return - epTitle = mobj.group('episode') - - mMovieParams = re.findall('(?:[^/]+)/(?P[^/?]+)[/?]?.*$' - IE_NAME = u'escapist' - - def report_extraction(self, showName): - self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName) - - def report_config_download(self, showName): - self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName) - - def _real_extract(self, url): - htmlParser = HTMLParser.HTMLParser() - - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - showName = mobj.group('showname') - videoId = mobj.group('episode') - - self.report_extraction(showName) - try: - webPage = urllib2.urlopen(url).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err)) - return - - descMatch = re.search('[0-9]+)/(?P.*)$' - IE_NAME = u'collegehumor' - - def report_webpage(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) - - def _real_extract(self, url): - htmlParser = HTMLParser.HTMLParser() - - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - video_id = mobj.group('videoid') - - self.report_webpage(video_id) - request = urllib2.Request(url) - try: - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) - return - - m = re.search(r'id="video:(?P[0-9]+)"', webpage) - if m is None: - self._downloader.trouble(u'ERROR: Cannot extract internal video ID') - return - internal_video_id = m.group('internalvideoid') - - info = { - 'id': video_id, - 'internal_id': internal_video_id, - } - - self.report_extraction(video_id) - xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id - try: - metaXml = urllib2.urlopen(xmlUrl).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err)) - return - - mdoc = xml.etree.ElementTree.fromstring(metaXml) - try: - videoNode = mdoc.findall('./video')[0] - info['description'] = videoNode.findall('./description')[0].text - info['title'] = videoNode.findall('./caption')[0].text - info['stitle'] = _simplify_title(info['title']) - info['url'] = videoNode.findall('./file')[0].text - info['thumbnail'] = videoNode.findall('./thumbnail')[0].text - info['ext'] = info['url'].rpartition('.')[2] - info['format'] = info['ext'] - except IndexError: - self._downloader.trouble(u'\nERROR: Invalid metadata XML file') - return - - self._downloader.increment_downloads() - - try: - self._downloader.process_info(info) - except UnavailableVideoError, err: - self._downloader.trouble(u'\nERROR: unable to download video') - - -class XVideosIE(InfoExtractor): - """Information extractor for xvideos.com""" - - _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)' - IE_NAME = u'xvideos' - - def report_webpage(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) - - def _real_extract(self, url): - htmlParser = HTMLParser.HTMLParser() - - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - video_id = mobj.group(1).decode('utf-8') - - self.report_webpage(video_id) - - request = urllib2.Request(r'http://www.xvideos.com/video' + video_id) - try: - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) - return - - self.report_extraction(video_id) - - - # Extract video URL - mobj = re.search(r'flv_url=(.+?)&', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video url') - return - video_url = urllib2.unquote(mobj.group(1).decode('utf-8')) - - - # Extract title - mobj = re.search(r'(.*?)\s+-\s+XVID', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = mobj.group(1).decode('utf-8') - - - # Extract video thumbnail - mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]/[a-fA-F0-9]/[a-fA-F0-9]/([a-fA-F0-9.]+jpg)', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video thumbnail') - return - video_thumbnail = mobj.group(1).decode('utf-8') - - - - self._downloader.increment_downloads() - info = { - 'id': video_id, - 'url': video_url, - 'uploader': None, - 'upload_date': None, - 'title': video_title, - 'stitle': _simplify_title(video_title), - 'ext': 'flv', - 'format': 'flv', - 'thumbnail': video_thumbnail, - 'description': None, - 'player_url': None, - } - - try: - self._downloader.process_info(info) - except UnavailableVideoError, err: - self._downloader.trouble(u'\nERROR: unable to download ' + video_id) - - -class SoundcloudIE(InfoExtractor): - """Information extractor for soundcloud.com - To access the media, the uid of the song and a stream token - must be extracted from the page source and the script must make - a request to media.soundcloud.com/crossdomain.xml. Then - the media can be grabbed by requesting from an url composed - of the stream token and uid - """ - - _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)' - IE_NAME = u'soundcloud' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_webpage(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) - - def _real_extract(self, url): - htmlParser = HTMLParser.HTMLParser() - - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - - # extract uploader (which is in the url) - uploader = mobj.group(1).decode('utf-8') - # extract simple title (uploader + slug of song title) - slug_title = mobj.group(2).decode('utf-8') - simple_title = uploader + '-' + slug_title - - self.report_webpage('%s/%s' % (uploader, slug_title)) - - request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title)) - try: - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) - return - - self.report_extraction('%s/%s' % (uploader, slug_title)) - - # extract uid and stream token that soundcloud hands out for access - mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage) - if mobj: - video_id = mobj.group(1) - stream_token = mobj.group(2) - - # extract unsimplified title - mobj = re.search('"title":"(.*?)",', webpage) - if mobj: - title = mobj.group(1) - - # construct media url (with uid/token) - mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s" - mediaURL = mediaURL % (video_id, stream_token) - - # description - description = u'No description available' - mobj = re.search('track-description-value"><p>(.*?)</p>', webpage) - if mobj: - description = mobj.group(1) - - # upload date - upload_date = None - mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage) - if mobj: - try: - upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d') - except Exception, e: - print str(e) - - # for soundcloud, a request to a cross domain is required for cookies - request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers) - - try: - self._downloader.process_info({ - 'id': video_id.decode('utf-8'), - 'url': mediaURL, - 'uploader': uploader.decode('utf-8'), - 'upload_date': upload_date, - 'title': simple_title.decode('utf-8'), - 'stitle': simple_title.decode('utf-8'), - 'ext': u'mp3', - 'format': u'NA', - 'player_url': None, - 'description': description.decode('utf-8') - }) - except UnavailableVideoError: - self._downloader.trouble(u'\nERROR: unable to download video') - - -class InfoQIE(InfoExtractor): - """Information extractor for infoq.com""" - - _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$' - IE_NAME = u'infoq' - - def report_webpage(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) - - def _real_extract(self, url): - htmlParser = HTMLParser.HTMLParser() - - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - - self.report_webpage(url) - - request = urllib2.Request(url) - try: - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) - return - - self.report_extraction(url) - - - # Extract video URL - mobj = re.search(r"jsclassref='([^']*)'", webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video url') - return - video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64')) - - - # Extract title - mobj = re.search(r'contentTitle = "(.*?)";', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract video title') - return - video_title = mobj.group(1).decode('utf-8') - - # Extract description - video_description = u'No description available.' - mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage) - if mobj is not None: - video_description = mobj.group(1).decode('utf-8') - - video_filename = video_url.split('/')[-1] - video_id, extension = video_filename.split('.') - - self._downloader.increment_downloads() - info = { - 'id': video_id, - 'url': video_url, - 'uploader': None, - 'upload_date': None, - 'title': video_title, - 'stitle': _simplify_title(video_title), - 'ext': extension, - 'format': extension, # Extension is always(?) mp4, but seems to be flv - 'thumbnail': None, - 'description': video_description, - 'player_url': None, - } - - try: - self._downloader.process_info(info) - except UnavailableVideoError, err: - self._downloader.trouble(u'\nERROR: unable to download ' + video_url) - -class MixcloudIE(InfoExtractor): - """Information extractor for www.mixcloud.com""" - _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)' - IE_NAME = u'mixcloud' - - def __init__(self, downloader=None): - InfoExtractor.__init__(self, downloader) - - def report_download_json(self, file_id): - """Report JSON download.""" - self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME) - - def report_extraction(self, file_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id)) - - def get_urls(self, jsonData, fmt, bitrate='best'): - """Get urls from 'audio_formats' section in json""" - file_url = None - try: - bitrate_list = jsonData[fmt] - if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list: - bitrate = max(bitrate_list) # select highest - - url_list = jsonData[fmt][bitrate] - except TypeError: # we have no bitrate info. - url_list = jsonData[fmt] - - return url_list - - def check_urls(self, url_list): - """Returns 1st active url from list""" - for url in url_list: - try: - urllib2.urlopen(url) - return url - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - url = None - - return None - - def _print_formats(self, formats): - print 'Available formats:' - for fmt in formats.keys(): - for b in formats[fmt]: - try: - ext = formats[fmt][b][0] - print '%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]) - except TypeError: # we have no bitrate info - ext = formats[fmt][0] - print '%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]) - break - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - # extract uploader & filename from url - uploader = mobj.group(1).decode('utf-8') - file_id = uploader + "-" + mobj.group(2).decode('utf-8') - - # construct API request - file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json' - # retrieve .json file with links to files - request = urllib2.Request(file_url) - try: - self.report_download_json(file_url) - jsonData = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % str(err)) - return - - # parse JSON - json_data = json.loads(jsonData) - player_url = json_data['player_swf_url'] - formats = dict(json_data['audio_formats']) - - req_format = self._downloader.params.get('format', None) - bitrate = None - - if self._downloader.params.get('listformats', None): - self._print_formats(formats) - return - - if req_format is None or req_format == 'best': - for format_param in formats.keys(): - url_list = self.get_urls(formats, format_param) - # check urls - file_url = self.check_urls(url_list) - if file_url is not None: - break # got it! - else: - if req_format not in formats.keys(): - self._downloader.trouble(u'ERROR: format is not available') - return - - url_list = self.get_urls(formats, req_format) - file_url = self.check_urls(url_list) - format_param = req_format - - # We have audio - self._downloader.increment_downloads() - try: - # Process file information - self._downloader.process_info({ - 'id': file_id.decode('utf-8'), - 'url': file_url.decode('utf-8'), - 'uploader': uploader.decode('utf-8'), - 'upload_date': u'NA', - 'title': json_data['name'], - 'stitle': _simplify_title(json_data['name']), - 'ext': file_url.split('.')[-1].decode('utf-8'), - 'format': (format_param is None and u'NA' or format_param.decode('utf-8')), - 'thumbnail': json_data['thumbnail_url'], - 'description': json_data['description'], - 'player_url': player_url.decode('utf-8'), - }) - except UnavailableVideoError, err: - self._downloader.trouble(u'ERROR: unable to download file') - -class StanfordOpenClassroomIE(InfoExtractor): - """Information extractor for Stanford's Open ClassRoom""" - - _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$' - IE_NAME = u'stanfordoc' - - def report_download_webpage(self, objid): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid)) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - - if mobj.group('course') and mobj.group('video'): # A specific video - course = mobj.group('course') - video = mobj.group('video') - info = { - 'id': _simplify_title(course + '_' + video), - } - - self.report_extraction(info['id']) - baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/' - xmlUrl = baseUrl + video + '.xml' - try: - metaXml = urllib2.urlopen(xmlUrl).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err)) - return - mdoc = xml.etree.ElementTree.fromstring(metaXml) - try: - info['title'] = mdoc.findall('./title')[0].text - info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text - except IndexError: - self._downloader.trouble(u'\nERROR: Invalid metadata XML file') - return - info['stitle'] = _simplify_title(info['title']) - info['ext'] = info['url'].rpartition('.')[2] - info['format'] = info['ext'] - self._downloader.increment_downloads() - try: - self._downloader.process_info(info) - except UnavailableVideoError, err: - self._downloader.trouble(u'\nERROR: unable to download video') - elif mobj.group('course'): # A course page - unescapeHTML = HTMLParser.HTMLParser().unescape - - course = mobj.group('course') - info = { - 'id': _simplify_title(course), - 'type': 'playlist', - } - - self.report_download_webpage(info['id']) - try: - coursepage = urllib2.urlopen(url).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err)) - return - - m = re.search('<h1>([^<]+)</h1>', coursepage) - if m: - info['title'] = unescapeHTML(m.group(1)) - else: - info['title'] = info['id'] - info['stitle'] = _simplify_title(info['title']) - - m = re.search('<description>([^<]+)</description>', coursepage) - if m: - info['description'] = unescapeHTML(m.group(1)) - - links = _orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage)) - info['list'] = [ - { - 'type': 'reference', - 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage), - } - for vpage in links] - - for entry in info['list']: - assert entry['type'] == 'reference' - self.extract(entry['url']) - else: # Root page - unescapeHTML = HTMLParser.HTMLParser().unescape - - info = { - 'id': 'Stanford OpenClassroom', - 'type': 'playlist', - } - - self.report_download_webpage(info['id']) - rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php' - try: - rootpage = urllib2.urlopen(rootURL).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err)) - return - - info['title'] = info['id'] - info['stitle'] = _simplify_title(info['title']) - - links = _orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage)) - info['list'] = [ - { - 'type': 'reference', - 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage), - } - for cpage in links] - - for entry in info['list']: - assert entry['type'] == 'reference' - self.extract(entry['url']) - -class MTVIE(InfoExtractor): - """Information extractor for MTV.com""" - - _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$' - IE_NAME = u'mtv' - - def report_webpage(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id)) - - def report_extraction(self, video_id): - """Report information extraction.""" - self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id)) - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - return - if not mobj.group('proto'): - url = 'http://' + url - video_id = mobj.group('videoid') - self.report_webpage(video_id) - - request = urllib2.Request(url) - try: - webpage = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err)) - return - - mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract song name') - return - song_name = _unescapeHTML(mobj.group(1).decode('iso-8859-1')) - mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract performer') - return - performer = _unescapeHTML(mobj.group(1).decode('iso-8859-1')) - video_title = performer + ' - ' + song_name - - mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to mtvn_uri') - return - mtvn_uri = mobj.group(1) - - mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract content id') - return - content_id = mobj.group(1) - - videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri - self.report_extraction(video_id) - request = urllib2.Request(videogen_url) - try: - metadataXml = urllib2.urlopen(request).read() - except (urllib2.URLError, httplib.HTTPException, socket.error), err: - self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % str(err)) - return - - mdoc = xml.etree.ElementTree.fromstring(metadataXml) - renditions = mdoc.findall('.//rendition') - - # For now, always pick the highest quality. - rendition = renditions[-1] - - try: - _,_,ext = rendition.attrib['type'].partition('/') - format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate'] - video_url = rendition.find('./src').text - except KeyError: - self._downloader.trouble('Invalid rendition field.') - return - - self._downloader.increment_downloads() - info = { - 'id': video_id, - 'url': video_url, - 'uploader': performer, - 'title': video_title, - 'stitle': _simplify_title(video_title), - 'ext': ext, - 'format': format, - } - - try: - self._downloader.process_info(info) - except UnavailableVideoError, err: - self._downloader.trouble(u'\nERROR: unable to download ' + video_id) - - -class PostProcessor(object): - """Post Processor class. - - PostProcessor objects can be added to downloaders with their - add_post_processor() method. When the downloader has finished a - successful download, it will take its internal chain of PostProcessors - and start calling the run() method on each one of them, first with - an initial argument and then with the returned value of the previous - PostProcessor. - - The chain will be stopped if one of them ever returns None or the end - of the chain is reached. - - PostProcessor objects follow a "mutual registration" process similar - to InfoExtractor objects. - """ - - _downloader = None - - def __init__(self, downloader=None): - self._downloader = downloader - - def set_downloader(self, downloader): - """Sets the downloader for this PP.""" - self._downloader = downloader - - def run(self, information): - """Run the PostProcessor. - - The "information" argument is a dictionary like the ones - composed by InfoExtractors. The only difference is that this - one has an extra field called "filepath" that points to the - downloaded file. - - When this method returns None, the postprocessing chain is - stopped. However, this method may return an information - dictionary that will be passed to the next postprocessing - object in the chain. It can be the one it received after - changing some fields. - - In addition, this method may raise a PostProcessingError - exception that will be taken into account by the downloader - it was called from. - """ - return information # by default, do nothing - -class AudioConversionError(BaseException): - def __init__(self, message): - self.message = message - -class FFmpegExtractAudioPP(PostProcessor): - - def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False): - PostProcessor.__init__(self, downloader) - if preferredcodec is None: - preferredcodec = 'best' - self._preferredcodec = preferredcodec - self._preferredquality = preferredquality - self._keepvideo = keepvideo - - @staticmethod - def get_audio_codec(path): - try: - cmd = ['ffprobe', '-show_streams', '--', _encodeFilename(path)] - handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE) - output = handle.communicate()[0] - if handle.wait() != 0: - return None - except (IOError, OSError): - return None - audio_codec = None - for line in output.split('\n'): - if line.startswith('codec_name='): - audio_codec = line.split('=')[1].strip() - elif line.strip() == 'codec_type=audio' and audio_codec is not None: - return audio_codec - return None - - @staticmethod - def run_ffmpeg(path, out_path, codec, more_opts): - if codec is None: - acodec_opts = [] - else: - acodec_opts = ['-acodec', codec] - cmd = ['ffmpeg', '-y', '-i', _encodeFilename(path), '-vn'] + acodec_opts + more_opts + ['--', _encodeFilename(out_path)] - try: - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout,stderr = p.communicate() - except (IOError, OSError): - e = sys.exc_info()[1] - if isinstance(e, OSError) and e.errno == 2: - raise AudioConversionError('ffmpeg not found. Please install ffmpeg.') - else: - raise e - if p.returncode != 0: - msg = stderr.strip().split('\n')[-1] - raise AudioConversionError(msg) - - def run(self, information): - path = information['filepath'] - - filecodec = self.get_audio_codec(path) - if filecodec is None: - self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe') - return None - - more_opts = [] - if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'): - if self._preferredcodec == 'm4a' and filecodec == 'aac': - # Lossless, but in another container - acodec = 'copy' - extension = self._preferredcodec - more_opts = ['-absf', 'aac_adtstoasc'] - elif filecodec in ['aac', 'mp3', 'vorbis']: - # Lossless if possible - acodec = 'copy' - extension = filecodec - if filecodec == 'aac': - more_opts = ['-f', 'adts'] - if filecodec == 'vorbis': - extension = 'ogg' - else: - # MP3 otherwise. - acodec = 'libmp3lame' - extension = 'mp3' - more_opts = [] - if self._preferredquality is not None: - more_opts += ['-ab', self._preferredquality] - else: - # We convert the audio (lossy) - acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec] - extension = self._preferredcodec - more_opts = [] - if self._preferredquality is not None: - more_opts += ['-ab', self._preferredquality] - if self._preferredcodec == 'aac': - more_opts += ['-f', 'adts'] - if self._preferredcodec == 'm4a': - more_opts += ['-absf', 'aac_adtstoasc'] - if self._preferredcodec == 'vorbis': - extension = 'ogg' - if self._preferredcodec == 'wav': - extension = 'wav' - more_opts += ['-f', 'wav'] - - prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups - new_path = prefix + sep + extension - self._downloader.to_screen(u'[ffmpeg] Destination: ' + new_path) - try: - self.run_ffmpeg(path, new_path, acodec, more_opts) - except: - etype,e,tb = sys.exc_info() - if isinstance(e, AudioConversionError): - self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message) - else: - self._downloader.to_stderr(u'ERROR: error running ffmpeg') - return None - - # Try to update the date time for extracted audio file. - if information.get('filetime') is not None: - try: - os.utime(_encodeFilename(new_path), (time.time(), information['filetime'])) - except: - self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file') - - if not self._keepvideo: - try: - os.remove(_encodeFilename(path)) - except (IOError, OSError): - self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') - return None - - information['filepath'] = new_path - return information +from utils import * +from FileDownloader import * +from InfoExtractors import * +from PostProcessor import * def updateSelf(downloader, filename): ''' Update the program file with the latest version from the repository ''' # Note: downloader only used for options + if not os.access(filename, os.W_OK): sys.exit('ERROR: no write permissions on %s' % filename) downloader.to_screen(u'Updating to latest version...') - try: + urlv = urllib2.urlopen(UPDATE_URL_VERSION) + newversion = urlv.read().strip() + if newversion == __version__: + downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') + return + urlv.close() + + if hasattr(sys, "frozen"): #py2exe + exe = os.path.abspath(filename) + directory = os.path.dirname(exe) + if not os.access(directory, os.W_OK): + sys.exit('ERROR: no write permissions on %s' % directory) + try: - urlh = urllib.urlopen(UPDATE_URL) + urlh = urllib2.urlopen(UPDATE_URL_EXE) newcontent = urlh.read() + urlh.close() + with open(exe + '.new', 'wb') as outf: + outf.write(newcontent) + except (IOError, OSError), err: + sys.exit('ERROR: unable to download latest version') - vmatch = re.search("__version__ = '([^']+)'", newcontent) - if vmatch is not None and vmatch.group(1) == __version__: - downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') - return - finally: + try: + bat = os.path.join(directory, 'youtube-dl-updater.bat') + b = open(bat, 'w') + + print >> b, """ +echo Updating youtube-dl... +ping 127.0.0.1 -n 5 -w 1000 > NUL +move /Y "%s.new" "%s" +del "%s" + """ %(exe, exe, bat) + + b.close() + + os.startfile(bat) + except (IOError, OSError), err: + sys.exit('ERROR: unable to overwrite current version') + + else: + try: + urlh = urllib2.urlopen(UPDATE_URL) + newcontent = urlh.read() urlh.close() - except (IOError, OSError), err: - sys.exit('ERROR: unable to download latest version') + except (IOError, OSError), err: + sys.exit('ERROR: unable to download latest version') - try: - outf = open(filename, 'wb') try: - outf.write(newcontent) - finally: - outf.close() - except (IOError, OSError), err: - sys.exit('ERROR: unable to overwrite current version') + with open(filename, 'wb') as outf: + outf.write(newcontent) + except (IOError, OSError), err: + sys.exit('ERROR: unable to overwrite current version') downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') @@ -4328,6 +220,12 @@ def parseOpts(): action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') video_format.add_option('-F', '--list-formats', action='store_true', dest='listformats', help='list all available formats (currently youtube only)') + video_format.add_option('--write-srt', + action='store_true', dest='writesubtitles', + help='write video closed captions to a .srt file (currently youtube only)', default=False) + video_format.add_option('--srt-lang', + action='store', dest='subtitleslang', metavar='LANG', + help='language of the closed captions to download (optional) use IETF language tags like \'en\'') verbosity.add_option('-q', '--quiet', @@ -4395,11 +293,11 @@ def parseOpts(): postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False, - help='convert video files to audio-only files (requires ffmpeg and ffprobe)') + help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)') postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default') postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='128K', - help='ffmpeg audio bitrate specification, 128k by default') + help='ffmpeg/avconv audio bitrate specification, 128k by default') postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, help='keeps the video file on disk after the post-processing; the video is erased by default') @@ -4426,23 +324,21 @@ def gen_extractors(): """ Return a list of an instance of every supported extractor. The order does matter; the first extractor matched is the one handling the URL. """ - youtube_ie = YoutubeIE() - google_ie = GoogleIE() - yahoo_ie = YahooIE() return [ - YoutubePlaylistIE(youtube_ie), - YoutubeUserIE(youtube_ie), - YoutubeSearchIE(youtube_ie), - youtube_ie, - MetacafeIE(youtube_ie), + YoutubePlaylistIE(), + YoutubeUserIE(), + YoutubeSearchIE(), + YoutubeIE(), + MetacafeIE(), DailymotionIE(), - google_ie, - GoogleSearchIE(google_ie), + GoogleIE(), + GoogleSearchIE(), PhotobucketIE(), - yahoo_ie, - YahooSearchIE(yahoo_ie), + YahooIE(), + YahooSearchIE(), DepositFilesIE(), FacebookIE(), + BlipTVUserIE(), BlipTVIE(), VimeoIE(), MyVideoIE(), @@ -4492,6 +388,7 @@ def _real_main(): except IOError: sys.exit(u'ERROR: batch file could not be read') all_urls = batchurls + args + all_urls = map(lambda url: url.strip(), all_urls) # General configuration cookie_processor = urllib2.HTTPCookieProcessor(jar) @@ -4500,9 +397,6 @@ def _real_main(): urllib2.install_opener(opener) socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words) - if opts.verbose: - print(u'[debug] Proxy map: ' + str(proxy_handler.proxies)) - extractors = gen_extractors() if opts.list_extractors: @@ -4592,12 +486,18 @@ def _real_main(): 'updatetime': opts.updatetime, 'writedescription': opts.writedescription, 'writeinfojson': opts.writeinfojson, + 'writesubtitles': opts.writesubtitles, + 'subtitleslang': opts.subtitleslang, 'matchtitle': opts.matchtitle, 'rejecttitle': opts.rejecttitle, 'max_downloads': opts.max_downloads, 'prefer_free_formats': opts.prefer_free_formats, 'verbose': opts.verbose, }) + + if opts.verbose: + fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies)) + for extractor in extractors: fd.add_info_extractor(extractor) @@ -4640,8 +540,3 @@ def main(): sys.exit(u'ERROR: fixed output name but more than one file to download') except KeyboardInterrupt: sys.exit(u'\nERROR: Interrupted by user') - -if __name__ == '__main__': - main() - -# vim: set ts=4 sw=4 sts=4 noet ai si filetype=python: diff --git a/youtube_dl/__main__.py b/youtube_dl/__main__.py new file mode 100755 index 0000000..6f20402 --- /dev/null +++ b/youtube_dl/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import __init__ + +if __name__ == '__main__': + __init__.main() diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py new file mode 100644 index 0000000..2853ba5 --- /dev/null +++ b/youtube_dl/utils.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import gzip +import htmlentitydefs +import HTMLParser +import locale +import os +import re +import sys +import zlib +import urllib2 +import email.utils +import json + +try: + import cStringIO as StringIO +except ImportError: + import StringIO + +std_headers = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1', + 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'en-us,en;q=0.5', +} + +def preferredencoding(): + """Get preferred encoding. + + Returns the best encoding scheme for the system, based on + locale.getpreferredencoding() and some further tweaks. + """ + def yield_preferredencoding(): + try: + pref = locale.getpreferredencoding() + u'TEST'.encode(pref) + except: + pref = 'UTF-8' + while True: + yield pref + return yield_preferredencoding().next() + + +def htmlentity_transform(matchobj): + """Transforms an HTML entity to a Unicode character. + + This function receives a match object and is intended to be used with + the re.sub() function. + """ + entity = matchobj.group(1) + + # Known non-numeric HTML entity + if entity in htmlentitydefs.name2codepoint: + return unichr(htmlentitydefs.name2codepoint[entity]) + + # Unicode character + mobj = re.match(ur'(?u)#(x?\d+)', entity) + if mobj is not None: + numstr = mobj.group(1) + if numstr.startswith(u'x'): + base = 16 + numstr = u'0%s' % numstr + else: + base = 10 + return unichr(long(numstr, base)) + + # Unknown entity in name, return its literal representation + return (u'&%s;' % entity) + +HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix +class IDParser(HTMLParser.HTMLParser): + """Modified HTMLParser that isolates a tag with the specified id""" + def __init__(self, id): + self.id = id + self.result = None + self.started = False + self.depth = {} + self.html = None + self.watch_startpos = False + self.error_count = 0 + HTMLParser.HTMLParser.__init__(self) + + def error(self, message): + print >> sys.stderr, self.getpos() + if self.error_count > 10 or self.started: + raise HTMLParser.HTMLParseError(message, self.getpos()) + self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line + self.error_count += 1 + self.goahead(1) + + def loads(self, html): + self.html = html + self.feed(html) + self.close() + + def handle_starttag(self, tag, attrs): + attrs = dict(attrs) + if self.started: + self.find_startpos(None) + if 'id' in attrs and attrs['id'] == self.id: + self.result = [tag] + self.started = True + self.watch_startpos = True + if self.started: + if not tag in self.depth: self.depth[tag] = 0 + self.depth[tag] += 1 + + def handle_endtag(self, tag): + if self.started: + if tag in self.depth: self.depth[tag] -= 1 + if self.depth[self.result[0]] == 0: + self.started = False + self.result.append(self.getpos()) + + def find_startpos(self, x): + """Needed to put the start position of the result (self.result[1]) + after the opening tag with the requested id""" + if self.watch_startpos: + self.watch_startpos = False + self.result.append(self.getpos()) + handle_entityref = handle_charref = handle_data = handle_comment = \ + handle_decl = handle_pi = unknown_decl = find_startpos + + def get_result(self): + if self.result == None: return None + if len(self.result) != 3: return None + lines = self.html.split('\n') + lines = lines[self.result[1][0]-1:self.result[2][0]] + lines[0] = lines[0][self.result[1][1]:] + if len(lines) == 1: + lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]] + lines[-1] = lines[-1][:self.result[2][1]] + return '\n'.join(lines).strip() + +def get_element_by_id(id, html): + """Return the content of the tag with the specified id in the passed HTML document""" + parser = IDParser(id) + try: + parser.loads(html) + except HTMLParser.HTMLParseError: + pass + return parser.get_result() + + +def clean_html(html): + """Clean an HTML snippet into a readable string""" + # Newline vs <br /> + html = html.replace('\n', ' ') + html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html) + # Strip html tags + html = re.sub('<.*?>', '', html) + # Replace html entities + html = unescapeHTML(html) + return html + + +def sanitize_open(filename, open_mode): + """Try to open the given filename, and slightly tweak it if this fails. + + Attempts to open the given filename. If this fails, it tries to change + the filename slightly, step by step, until it's either able to open it + or it fails and raises a final exception, like the standard open() + function. + + It returns the tuple (stream, definitive_file_name). + """ + try: + if filename == u'-': + if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + return (sys.stdout, filename) + stream = open(encodeFilename(filename), open_mode) + return (stream, filename) + except (IOError, OSError), err: + # In case of error, try to remove win32 forbidden chars + filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename) + + # An exception here should be caught in the caller + stream = open(encodeFilename(filename), open_mode) + return (stream, filename) + + +def timeconvert(timestr): + """Convert RFC 2822 defined time string into system timestamp""" + timestamp = None + timetuple = email.utils.parsedate_tz(timestr) + if timetuple is not None: + timestamp = email.utils.mktime_tz(timetuple) + return timestamp + +def sanitize_filename(s): + """Sanitizes a string so it could be used as part of a filename.""" + def replace_insane(char): + if char in u' .\\/|?*<>:"' or ord(char) < 32: + return '_' + return char + return u''.join(map(replace_insane, s)).strip('_') + +def orderedSet(iterable): + """ Remove all duplicates from the input iterable """ + res = [] + for el in iterable: + if el not in res: + res.append(el) + return res + +def unescapeHTML(s): + """ + @param s a string (of type unicode) + """ + assert type(s) == type(u'') + + result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s) + return result + +def encodeFilename(s): + """ + @param s The name of the file (of type unicode) + """ + + assert type(s) == type(u'') + + if sys.platform == 'win32' and sys.getwindowsversion().major >= 5: + # Pass u'' directly to use Unicode APIs on Windows 2000 and up + # (Detecting Windows NT 4 is tricky because 'major >= 4' would + # match Windows 9x series as well. Besides, NT 4 is obsolete.) + return s + else: + return s.encode(sys.getfilesystemencoding(), 'ignore') + +class DownloadError(Exception): + """Download Error exception. + + This exception may be thrown by FileDownloader objects if they are not + configured to continue on errors. They will contain the appropriate + error message. + """ + pass + + +class SameFileError(Exception): + """Same File exception. + + This exception will be thrown by FileDownloader objects if they detect + multiple files would have to be downloaded to the same file on disk. + """ + pass + + +class PostProcessingError(Exception): + """Post Processing exception. + + This exception may be raised by PostProcessor's .run() method to + indicate an error in the postprocessing task. + """ + pass + +class MaxDownloadsReached(Exception): + """ --max-downloads limit has been reached. """ + pass + + +class UnavailableVideoError(Exception): + """Unavailable Format exception. + + This exception will be thrown when a video is requested + in a format that is not available for that video. + """ + pass + + +class ContentTooShortError(Exception): + """Content Too Short exception. + + This exception may be raised by FileDownloader objects when a file they + download is too small for what the server announced first, indicating + the connection was probably interrupted. + """ + # Both in bytes + downloaded = None + expected = None + + def __init__(self, downloaded, expected): + self.downloaded = downloaded + self.expected = expected + + +class Trouble(Exception): + """Trouble helper exception + + This is an exception to be handled with + FileDownloader.trouble + """ + +class YoutubeDLHandler(urllib2.HTTPHandler): + """Handler for HTTP requests and responses. + + This class, when installed with an OpenerDirector, automatically adds + the standard headers to every HTTP request and handles gzipped and + deflated responses from web servers. If compression is to be avoided in + a particular request, the original request in the program code only has + to include the HTTP header "Youtubedl-No-Compression", which will be + removed before making the real request. + + Part of this code was copied from: + + http://techknack.net/python-urllib2-handlers/ + + Andrew Rowls, the author of that code, agreed to release it to the + public domain. + """ + + @staticmethod + def deflate(data): + try: + return zlib.decompress(data, -zlib.MAX_WBITS) + except zlib.error: + return zlib.decompress(data) + + @staticmethod + def addinfourl_wrapper(stream, headers, url, code): + if hasattr(urllib2.addinfourl, 'getcode'): + return urllib2.addinfourl(stream, headers, url, code) + ret = urllib2.addinfourl(stream, headers, url) + ret.code = code + return ret + + def http_request(self, req): + for h in std_headers: + if h in req.headers: + del req.headers[h] + req.add_header(h, std_headers[h]) + if 'Youtubedl-no-compression' in req.headers: + if 'Accept-encoding' in req.headers: + del req.headers['Accept-encoding'] + del req.headers['Youtubedl-no-compression'] + return req + + def http_response(self, req, resp): + old_resp = resp + # gzip + if resp.headers.get('Content-encoding', '') == 'gzip': + gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r') + resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) + resp.msg = old_resp.msg + # deflate + if resp.headers.get('Content-encoding', '') == 'deflate': + gz = StringIO.StringIO(self.deflate(resp.read())) + resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) + resp.msg = old_resp.msg + return resp -- 2.41.1