From 93defbdfdad27b88fae4471d45880dbb567df518 Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Fri, 13 Sep 2024 14:33:41 -0400 Subject: [PATCH 01/10] Move `README.md` and `examples/*.md` to `mkdocs` --- docs/assets/laplace_logo.png | Bin 0 -> 19074 bytes docs/{ => assets}/regression_example.png | Bin 27900 -> 27900 bytes .../regression_example_online.png | Bin 28501 -> 28501 bytes docs/baselaplace.html | 1070 ------ {examples => docs}/calibration_example.md | 10 +- {examples => docs}/calibration_gp_example.md | 33 +- docs/curvature/asdfghjkl.html | 246 -- docs/curvature/asdl.html | 243 -- docs/curvature/backpack.html | 206 -- docs/curvature/curvature.html | 459 --- docs/curvature/curvlinops.html | 187 - docs/curvature/index.html | 935 ----- {examples => docs}/huggingface_example.md | 6 +- docs/index.html | 3076 ----------------- docs/index.md | 498 +++ docs/javascripts/mathjax.js | 19 + docs/laplace.html | 88 - docs/lllaplace.html | 459 --- {examples => docs}/regression_example.md | 22 +- {examples => docs}/reward_modeling_example.md | 12 +- docs/subnetlaplace.html | 291 -- docs/utils/enums.html | 282 -- docs/utils/feature_extractor.html | 212 -- docs/utils/index.html | 1473 -------- docs/utils/matrix.html | 312 -- docs/utils/metrics.html | 139 - docs/utils/subnetmask.html | 469 --- docs/utils/swag.html | 105 - docs/utils/utils.html | 268 -- examples/helper/util.py | 2 +- mkdocs.yml | 88 + pyproject.toml | 4 +- 32 files changed, 649 insertions(+), 10565 deletions(-) create mode 100644 docs/assets/laplace_logo.png rename docs/{ => assets}/regression_example.png (99%) rename docs/{ => assets}/regression_example_online.png (99%) delete mode 100644 docs/baselaplace.html rename {examples => docs}/calibration_example.md (95%) rename {examples => docs}/calibration_gp_example.md (92%) delete mode 100644 docs/curvature/asdfghjkl.html delete mode 100644 docs/curvature/asdl.html delete mode 100644 docs/curvature/backpack.html delete mode 100644 docs/curvature/curvature.html delete mode 100644 docs/curvature/curvlinops.html delete mode 100644 docs/curvature/index.html rename {examples => docs}/huggingface_example.md (98%) delete mode 100644 docs/index.html create mode 100644 docs/index.md create mode 100644 docs/javascripts/mathjax.js delete mode 100644 docs/laplace.html delete mode 100644 docs/lllaplace.html rename {examples => docs}/regression_example.md (89%) rename {examples => docs}/reward_modeling_example.md (97%) delete mode 100644 docs/subnetlaplace.html delete mode 100644 docs/utils/enums.html delete mode 100644 docs/utils/feature_extractor.html delete mode 100644 docs/utils/index.html delete mode 100644 docs/utils/matrix.html delete mode 100644 docs/utils/metrics.html delete mode 100644 docs/utils/subnetmask.html delete mode 100644 docs/utils/swag.html delete mode 100644 docs/utils/utils.html create mode 100644 mkdocs.yml diff --git a/docs/assets/laplace_logo.png b/docs/assets/laplace_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..82c4ed0642b7bd34c277db6d5dcb9228b132375d GIT binary patch literal 19074 zcmeFZWn7ir6E+H)jf6DP-3Be)9U>*&AdPf)NH3j^Be;w( z_Nf3J-zs~Y`VgnpsD#z9>FBHB4{mk_FwEEQP~REciB%}}xRyW7CvaPG_V+!@av8iq z*kG)%ejkUER~(iTC)?P#BedcFn#|DE6hXwW*y@e?W3;G>YvBclW#bEh9*djSJ6`W^ zzS0Qfpg<4&X zpv1Gt_Lz2a;8CqqBp4kr1SOo?o5Isd`GHU|&(SPX_Kj`Y5h!;}LJ09jT3PU_294by< z@6Na8i7UQ9EG|v)IAtbZL`>(V_Cnj)L!`rNX*>^?HBgMVrMmr8<#!F0i1+9$aDokk zT|#q|({VvS!1@aRjhOrdiwx|UjZ0q~`r zxvL4ar=6|6i-4yv?VmFQz}N6^v(ZxjImPvrFs-(t3bmwzvpF?4D;Fy}tq3|bHMNkl znT5a$DVcvC2mcbLwQ_ZJ6kucX@bF;uc*^SFY{|yK&(F`s&dJ8f$pX$`aq+TuHSuJz zccJ@xk^inEW$t3?Z0+c3?O;z0zplwk2RBz?T3Yyx{`c$ebGlkv{GU77yZn1C@Pcgc zpRjSTva|i~+Tf!?@b3yJJ6oHBJHxLp!XfnM%>UQ-{@q824Sw_g>tX)B>7Va{R~11Q zV*B5+iJ(94VA4TA5JQlc5?A*`+|BmdAz1Ujqk14Nh746jY=l(zI3t>C@O(qou^Zum zO?*&Pd(v+=lf-D30kdAQv){v9C{)z7zpJ$KHZ5$lRy);rBdw_;bt;X6^FT`F&3#(m zfa8qCY?qC`^ph<9jlQg7|F`>RSx=Aqo(>*+uV-EKZCo&a(k*+bE+q#2-{CC1RHuKU znN5I*i}T-6i6$2@8cGeJ{qNwXr>0>IPyE0d9f%t7?+D>DK>FVav#gel zoq!51!u{VBz?uL5MZnVif3hYpLI`fp)1dUHmKiP^1K_WLw3HOW(Rv4?8n5GZA#8a+ zqQ6@iqz%>8;cZRP=GKsha!MH={rqmRF_69fBl!tKi$sE;`;PW^ce{W49wb9cO;i5H zqDH=p)2s)lCgAS2eYnnUc0#|&oyD=u{olrm(cvU0>eKbT#E&)kHI$zZA%7?nLp&@c z#ijnof`TjD< z{A2QojM5XT$p0+R3JMwd%H6@tb5892`W>PMOB1&o_CXxZmDh?eo&f z#=%1C+n>$Q{5w-o44jGo+?g`$e5YnGLot&FxyxgFs;YoQ*c+zP;_XHmRv;4f*I~rQ zfmf9Zyv(MKX`gF!>3_?h(8rYPyP}7g*>Jepo$0aYH|f#F^WWJucnLU@WlwX1oJ>dE zm)e8cb1=ww`gW#knEtP=c#5CkVe@p*Kf5b#(z;ahlLRuKbHTH~(oyIp?5}S&QUlK= z8XY6f&Ju7d*_Hon(izr{_~y)YqD;qP{UMSW2X7KpcGW}VYp&lsx zSMH;o2`Aec%RwH4&#oM^b$0i6za0;XI!sg;QR&JGO8^ z*4Ci39zI_CG0dRJ{S9P}E?(%Rq31F(nj_jp$>)DB;)&KHIIbX-i;vrWYByPKu-=#c zG{5@|z8mn4d^ecde@j9wf@r<6Qu9KQD5%Bfj5I}wh|Q4var?>6^k`qMs1YP9I(y;2 z9V3MnMk)lqOswVbJ$F3*f{?EgLF8g2rP7G?bhp~qm?+Wc?& z+JlhnFb&isM)LlxN}ddO4_&$;FA>Zyp5Pi3q&(2@(E0`MUUGhCDTsa!lioiU@4*BQ z==@ZpcK?@}vaorb?exoXqc%#tYEeMI-> z#3#M}i{lNX?vdJ8T2C$dh-+LoN7@DcXWz6c!T{Dwx_MGr;)ywExxzQ?>gO%G;;Aas zbsDn_+;sn}Q{Ec@Yp)8Y%Z!;5nUr}lc^!Co?dMQ!%>g`w4beWPO<;f+`JE(t0YB>t zy-(ElNw5AnSZK_AJJf&zTDZx?J{PmDC-rhSjj=)Kf`>l<;?JS!Dn9@-BDTSFa+8V z;;Yf}8QS3geVB})eK`Y4}L#Fwu+ z=W&D>3)=6!`PnQXFV2Zaf^J)8tOlPSl$Jk^gg+{3vxN&i~ z-6Y6DoNaqn%R+x&d}qq#s$cZ?u|*?H@*M{(d@`PCGfm zK0gFBQU1NR6+ic2%~@${L1;v#{PG)FN{)Du%VHsAGUEj@6I2lNxKOYunH8eLx-Kp@ za(R1GSYAONkOsp&oSqS&9!K>KdEn%undnlj3K{pPtm0@w7EWy8-ztAs46l88J(S3> zvet6%Rcbh{MDWgB?b=~6&Hxr zR2IXF{qNz}REBu>P=c_0_=;Tj=B9lQzYz_{w*198PgE=ZaU}XDRFP*g4GG1ZRzuXm z-H(*+11~3l_c4Wt4D3?)qdG`cGu3tmhKN`pDV_aXZ6l)?a_Eg{=B z;eQ~!{f-H2v7}Z$851VYS@?J=-TtwRQ!7~6l0Z=0^2)6@Aq$wm)@5( z9wZ@H2;060#ullJCt)`Zgf>8gTW~>k6=Vgej*PZYA19rH;Z}*1xNcAMH=j)Ct#20P z#+8)Zz}M4Lip{9Cc}eG)W&#olR>&!mY#6~4%_+BQataCT38la34joOo6495ML6KM8 zEm21T?|qz73b(j0vNfSQJNKVxl`|)_x9RF}2~|C_J?W7e`ZT`WX4jEb5&}Gi z>IHU&7~fuB;4PI1%i9os`CiEj|XS40Ru!BjHVB7RM61T7TJG2~_5!bWZY zD`L6K@w?F^S>v7r8u#@Cht>;5P6|RS?4Ls4HuLRf>)ccBp*gNOO{&7pV~H^jht8f< z!&|vyuQMDQrB11jcW1$j=s|>>Ag4s2)QMi5Z6rm(!C^NQTVbHjN+eU%O1HV0+*TOe zEPkd0rl^(jxy`{EmO}~Vr#gOF8EK;SW$}+H-JMQb=8~jH&&Of-`_bECDHLgy>vP;( zo;;O}BR?$tU_Ako z@CbEvM71>ehL#z;3UPM{p;Uz%1|qIpE_ezxvnXLQv|nwvm)q@{^6K#@Mk+54+L3A{ zp*GLkv&DZ72<~&?Gd_QazP`PSg|@%wcV-~yN7!5J^jgbjcF6LNZ3LZzJnEYzfi}J! zGHJM4krMKRN%%@j=*sXw3{E@l^5E^xpe+j7=LbwUY+wAaDFn>o5t{gg3(d?p$1%Z% zgaH^V@7RnrJ>zlxrBdUuzkohi*eRWN31vqX%am!{t()x|`>57mZQi>x=zmw4ENSfW z9HlaJ^ynkiBRhQfp8kf%%Jk>MJg`@jtYc33AfI2~tzS$*&|);D(A0y3U^?A*1ov&|}^$^&&jr9ObFQg3wjXEj%6sqK<_HhBQF4=G{dX zIvWm6&UdXio3BoEpw)7L=Ki+a6$VWf9&B+4vJto=U8)a6zINpWz_(O|2!hUmha|XA z7^2bOww>Z&2eVf~y|;)y^AJM9Fx2Q`>2h&o-sC|c=*~sPW6NIn75#A>=0n61`Ii-A zj^c%|KY_nQSuCRgr)%_sVvS@N91mM-LBZi3Ow%BA=zC~o+J%1NhxGv{I9PEZQ&;{4uo@@iV1q`aY+?nKaLkVir@!H%8)%+k3PKf ztywSLrl0mScno~dHgce(fA2Ohz=kRJVj_Y1!dCg4u*{I9}Ht; z%3B*VGC;6mRLT(|@#bH0`wXhSA%U4k5FG}1@+5fo=!PyiWz~Rm%%~ngmV%SB7~C2h zX5nI^-I1mri#19$K?z8T3UkF?f2LGvkd?wm>?7D&*Y>cibK>#Fq_?l8Xs%mOHUUF?Nrq zs!Z+TZHwM^Z1B`uGOB9bV99JYrG*X0QF0GjzWt@sQuz_8TD` z14b=h3K5PW9+CLG0|uhIXP$up7i+0uxA^+n(tqP(mY5`(Q;~9R-7$YkfxPi1*AF#r z^7p4vV!LR@^>sdHdmWKFO5a?mi<$KshMJFlJdwucn+mA*{cJN{lGj;MNAV^0fgwmC zSOri_0z8S;>?dfMFaKbxG?cVUaC}=si|a=Oc8I!mle?W1%PwJ%Ev_TV{(DQ+46fHF z-pAL~Kc5YZg;RPLTLhbuq@u7?Fb>Tk+8(!F?FsH%_`xF$BJMdskiJh*wSGuHw&*Fy zU=)Ntf^;W0d#Gs|xna^MVE%a~Pk$Y<6kV`d__I1Rua%C6b{mEP#lr1Tzveq=`SFSV_y%0yA=bW z3u`HS95p}e1hKmnb4BJS!a`VD`qMZ%nRTk^u{*6xM7(PUe5pEi7Sp!ZdQ*uwI`;Z@ zp{lSNbSh82h-b$Ht}`hqDR~Bsb8#&zDJ*)=F5`rkgwN*Gstw5Cq#SjG*q-eQyo9Lq zD=mDU#aH#nw7oxDaPo9Q8i5hfTb*Vn0u)72+7brhKT)xX|cQD1iPtCFlaeS8VEC(cDHc`*(>d-OI2#s}1WQx#**nNL#bcJUpisS!h2SI^1Wya~ zTYVdgJ8NBu4{R{2TFAJp5-N;IK74uwwIlENRc_D}wcc@)kK6T&GJXNjhU&RutRZ|m z3mrRt9qb^*?N7L>(%D`3?6Gf{axUzF>=4>JSw}LL3$z2^+9m2p|cX|34+3$wH_WOS0QHyy1RSI{T+|V{49tgO-*kBm#cO$IN!wpOK(IMGiuUxBYutOQ| z40D=ma9RhMusZ=)xKr&rLuC!vNbQ5yLxc+rP6g~-wqF2oqIRx7kO^!@YfgrLgXN5v zYuO)*!GHIQ>>oh+L6bm26IV2Gce{VblgeuF*pl4+`lAKm(on18vAg}H*jFDXQ0aFm z8-FoPU`L6wc#PrAuUlGqYEz3XNGFPj71Yx*wlPV?Np}6aNVFDd`7<(Jk$ra@MRxti zwj%@2)mYvrXujxquxNxEWfs!hBm$OjI2J{en=NTFj%PfTsNtBpzRv`^-8?8CqGtLo z!yA^>JrBxcKg5(BtK9*YTfyxK!P(#rjGH_$n(VOO1Id5e!jRNHYJErSRnHqbh62X+ z(p&`DZ-CyLJ{cZq;C{{!t4((F@ayvTO^;RC;)`YMfM2|Q9d%!VQNnG?wW~cg9Qx$xmrV>WHQt|T*7Y`Zn5z8Z=PgP6rr{e|%#@#g4(tox%L}yI zy+gkJmgcNoZ#Wwt#0TAC{fhfIAFrb9bayuM=IX4Q9; z;7r2K`p{A%l?4t0g^92`9f5jC6E9Q+4{_v^_-}VQHVJYGSN(oE>+8DVTaz^~{JR-|)PB4BKl_aqf}#Npy6aExg;}=RM((zA7bio1Zyb4@w#q`0+%JS-G3= z6zI7q z@z;eI}Xk550Je+nmw)ws? zA_7sFJ3P$pf3{rTdGx|z6ObaiUuqbGb;^#%@w*TYOvGn75zy>T9E6 zRWIhN_&RvQZiFG>GfucW%}%ubKXM%E1_Sj3hJ3~{$MWV$DaOE^2jz`h?jFy~@ zdQTeqj-_yEzs752%*dtuejP&7=7M5Zs@a3Ne_iED2!n>Sm2CW>IA!B_lzHkY@e1mQiA$215g-$H*S>Id;}E8$wH^wWyDc|Sofs(V(Z$pa z<4)bI7wRBYPkQgUwXQR$Q~~?x-tX;=<&{Iibwzqd5hs~A91}{VEt+C`AMf4}m#1k6 za;@s_uDqSqN?+CNhrUfwh?w?a*UpOfaY`PUG6U%Y2gX=-GGT{+xBghw;`c8H80^{z zPgc9=C4FMLxb6Km1P)uPX%jRL>vnJYYE^nW-dYWDD=wm{CaniMB?y1$%unf^sr^~z z5-B!;s!spaq;S{QdcH!nDq&7Re{c01_gMz#<5yx&HkGI1*?HN|MsPCTb8#9qTxea2 z>=xO#Aylgd<+Lcs6I)*;=T$Lsx2}7i2q?10o9Ifk-O|q!(jOFU^1Xp6%WroISW#x} z8eZ|(nhpOH%xP6>X{eCDB32jO@~wTa^12-PvIi+%>plh7hlbf<+k3IO$5m$Ka1b*{BeE0g$dG!7zVoxF=&F<#%2db|O%Cn=@-&bUoC_Iu(!_T@XlQLN3e6qNX zLyc@`?O$3GN6H!)i9=K}W>Shp3afIxPOm6T_EuS69Xob%avtxtSj^=PW4+gS2YanL zvjV#eY7Fs-X%1(abmrFNs&qKc8N48e=_X8QkH>*Z^HETBXagXcQhd`poz(@3urBiz zy^-8zUL|R@#b6A?n+hA=tn=E7J|6!-8{z4&BTmY_PL==N?oLKD7|OThGy6fLx6bMQ z>u{rthi&Gw4R(y5Q0Uq&)|-2-*=d+<`SN7xp6z2_cGc%qqezT z2*>4G9v?#(ZGFnK|0F*E4nQ`_dCW8e%1Z~VKIfRDrYOGwMjuziD<^wekH`@3EP5T;-eb zArPmlyJfI9d(hwImk8tb{${|;jFi8}h<)P4>FWMMSfY4s)DsEVJFO(H!bqED2o_NeZ-7EHg!^1MbUw&K6p*?r6G z_ipzH-}w5H^aJw!0@;A(-B2_$z89Xzt?YJbMpvr6cS-?Xa)+kG?jjpgHZ9f7J8Q`n z?7wVInp9^_nkbu(zu*OMMOLVxZQ&b)qB+EuqjcN4zn&5NRS!aey6U-#g&C_oh#=bN@Rr4MuFrsQp|5gQ36_6>_M!(@flxv8Gu@ zrUDnC3g|CPg%in#*k{!bCtsxT^s%*QlLw*bJ*mi(yStXyXy{gpJ`?948jVBLjIR25 zNN$p!JU7Yt8Yg3q>;9xq#zLLa;xEgj&Lq4kK^P!9PMC1?B37ffZ%tPG-m=v!;%Dbp zss_;4?g#l;f9>Vlv-BTZMO9{w4M7uhl8t8%H;!XO4+;~tw^`#WYB1GXZX~be`iXg~ zP*PvHQXdnKK&)xJY4E|EbqGVV(^#mC&E)FZ_IV@W$f+F0wq(toq;}lZO(A6|mJ^kC zLE4C+uEj%N%V68k3&7kwiBFx;ebQ-0)FZ9sw9nLOTJy}y-97vfkG;i8Bm(YYLQ%KG?%pH!|51$AIU zOFQSNbWo8snzWo<{Ph}p@)n-stKjT?n=Gx?dZ%)awgo}bDQ87h?Z<0049tVv*p_<< zGX6J1JRq^Y#E*EeaxFnZ&oT8)W%{asusgc;)@o}MNp{K2cII73(&g{!qZ(&}d^bts zsO{->{b*EwCDQuhoRDuYXBaJP&GK=(TQr^2RJHxs#j7e&<|&s23JRMq6>nY6L(dOx z7o`2VTm#o`8=l~!8Y^Y0x_?4_X*I(>y+nHEBjz+iJ5RNa1UKOenxUw2U89xumyOAp zfW;!%t@nS>2gDiCb;y2|wy-|Xn~iqYb9d~jZh1IEc>3jYWCoxs(m@|IM5E@c)@J;N zwsQBWkAkUb+g)_qd9>61jZ=kO?|pGj`g%tnjvO&uJN1YE_C$48zY1 z>1aafiK5r<#d&}PBtm0S?>pXuShrxVXd<@Ly&pd<6|}hlKf=7Jr{BteVk2n_Npbw6 z)M)ZomsUvQOc{;3*j>D}1N?0B+bvB#@~bA5>ViJZl$2?03y8&ZwYhm+UMIkeEg9

dH|*EvAli98G+7+kWPutyjh*dkkoPDCM?~ zN;a>q%#sBtblvu(O>-8S=m$B^`ZIZvP zh6Y7vyW-=40@F0+(_GX7}+0ugiE@t|I+u z!j!akFCY=^4SjaGO}&yxd$kwqvX3VsulVC|!r868E;8C~x7&DJ*83u7VOx=6I#5DZ zeX9k}RZLTs*o^x`tV!cCbHU89c{+g9o7*Y@I*Q7Ks65f!bMkHsC-H2`H+0kyxsbq>Tdc(Bs^`WlRmqukuQ z;UOetFGpr&Z)Cn+&7RaJGwiZ~1!TQjcEF)5B-tqP!^x^M`m2#<9=QSQf-?BRUVb~|1ng<4mAa-4eJ~PK7X~hx4v5cL zO0~J2i=YyBlSi?}-5Uv67`OM8k+3J|5-J=JAi1*+o| zRCNiE>E2f@@Q%$n{!%@Ryof_nqY^RMVj#D)E(&NFLeQNs`ktlPFZi9!OPRu6R*=NZ zXq9Qd`PHDSE@M3P+3i)?4z~XdiI8XIbiG6FV)=(f895|d^z7Fb?~zJSIy|6X;7Hf%2~&kS%Tqh@kCR~5DTV|uS+kaCUx?Tt9Z7~` znZF1uxFa4iJ&$G&cW1GlZVPDJ9C;s2Ly-2P0?l}Kop9;k=mg!-$_Pz%)k;d|5>SS( z!n~Ovs!%x{*%?WuT4Qy;Gk@Dic_-PQh=2Bt8s}rA3pV~$)O3yItU@B7>)iIMg+Ko2 zSJN3&i510Zvi|ZK8Lll_FNha;=IWk;b#=0BImBnKx}9dlPN04oRuJ_44V`S1?r#CX z9_vEXc##OPmk|3{_m$vJNImtvo%~+{s~Qu-%E_YyWT0CL)NgP$CN)CxWYCkIwNKw& zwcR}hGOfx`jW!MxyH(k7GEKT-`vO}kPO}Jc!OaIdz2;+}|euhPqP~aA`21SK7Y`It5eqwUlt8~Vn;qf@x_j2|8fhfGzd|pYL}U zxp!5VrN@HKC;HH8g+TR4tF@5_>%=U*pFF#cD7r#^61AO6N7j3}e=N479hw6DQHHd| z?}IVKas_Af)pHTz zunV7GUv5{Fx^voY9wd!dzZ%CWHNlOL24WrN=iid$?w3iAHIZ%(u7qs}>HV+=z1K=BG)9aF`97rwpX`fq<*zd7uVjYc9OOFq2*x3GV`y zPPCGO#sw-|vyc^kstdj5O0b#{0c;Mf2(#dE$X;C{8MjT+bgQ3_sxVydQA~kI`OcBY zIUM2H@lJ?7ZE62I>F5z#O*`OOU*VFzdxaL|D<>kM$aNfMUmS^?g;vI#c4ulkEI8c? z?Q+$u^c_I>#lscLRMPLVVX1k^I|*Ku0g6A?kqp z^ozf>KPu-X5YrHm@Zejl19iq(bhWWceBKXuwlYmCR;CPFCt}rS2f`b!di(hjd6e(g z{lOI2L7WxEXwKP%Xs9sz*Bmwx<~U2?r@d@#%=~YMX!nGQLwL38Y<0fJ1{V3l{bmOj z6cH@7NPV0TR z$!0f^P#i2uj1()t0m@8Oi|W@-IK_$Qs!BBY!A40Q9*W**MPcB9Z>O&H*wfLpksSFdy7mp^;**v9k z<39!LGaLZ9H%rgxe}|*=fPR~|$7T(9iRX}N`dG9Q+|rZ%#YG?*g0XX9q(@$?v;35C zyYDzv;rW!uGkzE-A%eDW#LheP$m5kf;#Rh@K2>Ytl6ozz@^<1<<1S3CpXsShPrJfU zxu`%xyq|aDGy<*qVpx`nKqemHY?bP63eK8_+BI*2jCeRv05bnktte{i&&F{fHh)Iz zm7FOB8TAyRLv{h`s$`6Gvv}M2%KfV*`D_?EPr23V!>rXzht@}2iuY20wvkYT$!ioi zs-O{PT~*GMLA>;&9gY}Qe1+fny^hRGX3q~-`@3Sv z?&tnlyCgBtT<(j+XL{x_X3|?}(%BbqGZ{c4G?}=#Slqc@qDrHG6bkhd9uNEM}3@~1~0bg1_SnfRaNfGUtSW=u+a}5*$ zKu7%A*vIr?*SC(hbfheXdb+x{K6Sv@r`%t`RTi7$rKy{9jWwG?`N8lmhPG7-Oe1Pg zyFH(mc1Rug-m7<;jJ23(71 z4`ofExwuLVxFGVY?|5TV0k#+w>$J=QTN!o= zVLm{21Q5)MyTsQ+wo_kB^HM?>rKj0>Uo3=|1b z|DHWUOa%3zlD^3LBTcqQHk8(MtsK#Awy1XKKP@u&-+X85VYk@6#-oL0LC&>YDH{VspM zW9&)a(gJN=R>T49n7Y5(od~fmpe{>2M2p&5_%&S64@j+NC2z!L9~rV_*UYooXVoZI z{9j84<_GFFm|z}L?HAyxT(2gWJMkWW-!Wwa;pL|GS>EERX~9r&q)E^0rI% za}94Ax;^v!kG?ZS#*ZTYdm$BN$dtRs?#%eWLl{6&Z+C{ojTA=K#L*Wsk0e5p=-Ok6 zIV2$ZkCg2GsmIdc@r|K9X1j+o;W=yyk9!86>8+p{(Y!Bv9Yh>IYCV#5}10vj-x zu>dbELI=G~V?r8YjF7?Jae!P>Zcf46L@wqz%3s4j$DuY$PSndJpt)_mIn}$`KU(Wu z2U7?lvIGvFfksL3?)C=rStKA72=+0~zoL(U9%%Hbv$Bx?wdXL;2GScc_{haEJd+Br zzdF?Xq?-=>Jq_L#{|!X+zSoD{bz(ivEODCNfT66alCV(B z>cZiS`=2Ye!}V&wL+x--IYGhA!`60P4H~;j)LD%#8yP@SJ3YeVq*g<@9Vp4?>ug$%a&=1fMgPy@^|Xdkx1q`#37m1>3m>4tuw8xTFpd*9r&1%)korMZ?EtjVcTj z5a#W@#WvAu4p=Jl9WCl2m;|v{`;q*nTU}RNLjOIq)KO^rWGw{h_4AEv-*AC!=@1?F zqraf108!G+0c$Xl#qZLk-{L(k^m|A$?0sv~-sj%B{f-{0Os><^)oab(!=E?$;JWqc zpzvm)Z|Z)%RA=6sf>(r9qy#h!aLrkNG+V0`sI4|^m3B2G^lcuFTK}r*rhK~A)#1d{ zqFlsl^H(60rwi3~yO>|RF_^8SGy$S%mvs>1#|5%y6oU}zJh6oi0x#Cs0<2%n0~JG3 zu|}M%>gxE&n@A92B;EBq*Sc8nR#x7Yh_`@7Z#od_Cb@k8aWDhyX{uWp7SN8v(_hvg zlBX8LA$4r6=N&v9sk=IEUK{V-Y;ghMq%&sFQSI4T`#{6dF9B!cwd!C!@IT((R5rHK z`$+dN9?Bh`-o56_`3Ly*n}1*9szMI*Y`-?Wh2{T=2Ql6X~jwH(dl-Gz&h|6*Bi&g^&V8#K_(7iFIf4rw4A6rvFyiRhWHGL^luKHk6G+s9BY__M7l zScqKupxs`{vjpfA8RDQozbq4eqn1%q^g9iZ;U)1MNAj1tK zLX5*$oB1N`mgq<)8HT}OJzFG|t#t)*og?|c@HU`7O=4;0FF1{66?Oh8SL3pwR1{?h ze)^dD0925(GnQ?PKc|*t-=ojUW$CdX#_i$JWYIT&(yitEX~e)g3+lrXc04#P03H5U zKEUgy%~RO+MjTPD*T$K4P~9$5-D3?%dCFS9%zbmQ5pZiM_(`4LZj%^9{TGnQ3M01p z^HP?PugLG@9Z>9dnRUhrj3q}v&9A*FxHT-AT_XgUV(U+TTGSluraq6_2NY=o5EXfp zgE1*bQytD?W0x@Y?#{WDZ_)iQ>(Rd3n=1x=WW-UR%481$kt`D=-&dz95?O31uDmNO zC!okInzbVNv^;+Ks~}#pFu(IA7xr|H-q8QT91YLBPx7rKYQ+JN!)Cxa?1dnyBqmsJ z6Fbr3)xsG@zU55qD@)yadpr2Z!&Nc^eC)BYblVS#2$L7RJ&d;-FdFflm<0F!toxUYt#LXcNW{)yq1Dc`f{(o=bqQ?yJ9N^ zoM;7@Zz#FWyrkl_vY!DNXw9lRzoF-VQ!_0Bet8fXhnvaQ+)?G3VD88ow0fLxv-9p_ zR{)QRbJm+_NSnL!xn&|~l<7N=Dt>ofJRgv|+(b?ON6y67r>l#O=ECa{mJuNu1R7XJ zT%+4M!~~_QPjp%(4P}FLDWhvHQ1p~Pw0(MevVxb1h3#3&J<;m}Nz2qPL8!}-?l{GF z=ka$_d*LHb32-)JK_x(dcHr2vZ?EzOhe(R?{VuaPa2lTX6QrdiG2Fv1oP&2jbchVY zVZq}a^S0u^l|qXSOb`>Vbe_GH>W4}|O;@?UJE`t!e$MsO2UIExZZn`A`DZ`i(QsNH zMgzeKX4@q*$=%K3UCWWq<(FZ2m*)d@;G+=Ufp2SLCzI|FL;Eca0gU3~8>SJ1)qe9?ADT z+KH*;eSsUqjMKiFoInARsoEyV$4u$SW+oP6D7aSk5WeTQ5`w{15B)h^wg7Q~4SMusCzIau>+-I2zY$dXV*Mn)+!PLzG-4qUpbervBDEf$jFYcZlRFhuZGCMpib+7;7Qn(?sv(FA%&kpqMt@hM}%qz|-iaWKwJ zzw;^SbCC0*ii=b0GwqRn(K(#^MmR)<5;x^Nwz2sgt@0Uy7FPl)y-BEu>4T`D<*ew- zm`*f(g+(+&{m8lqCcd}6`1C0f%((yxx>ii6R6+w{vYQ(nnh#~ZC%$+24Cjg=Q~?Xy z2nZ#>IPU*B_eLKWQ9>--h*$a+;jUG5HE@z|fE>Yyq-c(+2weD6K_VVnoxUFwX_M{{qv zUN};hlSp)dMA=q6`51_bIjV#aKR&lY$|qWW@Z~L+yh#T|Zx}o?exg}+fb|6@>(s-4 z!{b?k8&cz}reFuq;S zp@kqLsHj=9#w{tN0f;`#KUB2c3RKqGNY`Qu<+~S3AS~k{#*ZmFjUAb#YuDQw3^&Tp zhgjgyEa)sSYek5cM#>P;AhBo}9>06nNFOBlTJ+H%F?LqH-K^$IpBK=YC@T2KBA7rm zdYiW(FYf_*8!jMA(iS!z@S;9=QivgmTheV^TYz$IQtPiM>;z&BL zkF5p!C3XR2w}TIj+0|^?R1Yn zioMyvxcC@*x}B_LSS7h_j7_4BbFRH|3qptCdCu~IWVv`8om=Xveo&pm`N49&1-AIc zp?xm*s}w)1dKrS;r}* z5^dhPjE0%&?bJP87Y{!^n*6B7BpJh{S$(a^JT>4E<lG8?QHcGuIIlOXd~S&1N-`^}(r9hmsa!(LggR7gnmgMFUzZ2w4sgcIC1G zdo=c@L>!LC5y}YNOx5~a-X@!6pec3wg0$<`*J7!r5S08yXWfG1p5F7Jg^T1W)9wrw zy}I+=kvdX5D}zp#P4&N3C?&*x19h-`F5v5{g!6Rq-mpe0V|)yR;r;Tf zM)d1!^`i8N3OrhnrnW2lAK}mTG^xr&%oBBOIJDi=?bfm(vMxSVkRfW5KxIh2#2<3D zx!DjC{u3_*J)ctg5&t+255r;;HO#i9wMV)xqw$%FvXkS%_rkKUvv|B8Z^V9Ofm*1? z-?&u|n69m-6D07+MEzC_ZyBZH`HR8v$4o5CbO#T<-A`g?+mk43y}KT~i+P^9AtJMX zEFFk2xlpO&bY!vdD(Pb96PMFP7VpWQX>!{lgqYq=2(4C>H6S{rt3F%B6MKP~L5nhV z*W?_dZp+|6UeR;-)F;qH*9@*eU@}BmlU(Fe2kto!}R=y zZe-E_&l;j9NE@g1+yqW9Ci5D!FPu3D={>Cx z*qpiLd1SgyWYE>R6@|do#~0v8l6U{oC`gAE6t0c4K%0PPExgtVt;0dqgZ9^7N?th) zA+#V3IH`4&XRDtItc{?+^b&Ytz;rq9tw=(q%D_%V+KN&oSRD`Cci0b1W?v#!sUn0{ zNCJlzO`X>YLkGA(8ChU7UPIthOl7HgH(;18ZOaH(DBBv6=`QrEK6>0-AuR j(z!YbA#|nZz<*}bd}T#Vx1Lj=V~9Lm{an^LB{Ts5Tl7zs literal 0 HcmV?d00001 diff --git a/docs/regression_example.png b/docs/assets/regression_example.png similarity index 99% rename from docs/regression_example.png rename to docs/assets/regression_example.png index 7d1cbe6eb64bb32d4fca89ec5b5ab5a1c2481ed0..1a630a3b5c0a0d69b3f8ee77c62ef20da0f23464 100644 GIT binary patch delta 43 zcmex!lkv|@#t9yBhB^uvB_##LR{Hw6i6sR&`6W4-NqYH3>H3!?-zjcPo0bCrj1Ljq delta 43 zcmex!lkv|@#t9yBMmh=^B_##LR{Hw6i6sR&`6W4-NqYH3>H3O(%r`ftP0IlQhAI&) diff --git a/docs/regression_example_online.png b/docs/assets/regression_example_online.png similarity index 99% rename from docs/regression_example_online.png rename to docs/assets/regression_example_online.png index 3033b067f258939d84fc5fe557d0846737e3da28..834a41ea375cbceb1abf088212625c830372db07 100644 GIT binary patch delta 43 zcmcb5kMZg~#t9yBhB^uvB_##LR{Hw6i6sR&`6W4-NqYH3>H3!?-zjcPd!GjYfwK|w delta 43 zcmcb5kMZg~#t9yBMmh=^B_##LR{Hw6i6sR&`6W4-NqYH3>H3O(%r`ftz0U&xd(II= diff --git a/docs/baselaplace.html b/docs/baselaplace.html deleted file mode 100644 index c700918c..00000000 --- a/docs/baselaplace.html +++ /dev/null @@ -1,1070 +0,0 @@ - - - - - - -laplace.baselaplace API documentation - - - - - - - - - - - - - -

-
-
-

Module laplace.baselaplace

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class BaseLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Baseclass for all Laplace approximations in this library.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
likelihood : Likelihood or str in {'classification', 'regression', 'reward_modeling'}
-
determines the log likelihood Hessian approximation. -In the case of 'reward_modeling', it fits Laplace using the classification likelihood, -then does prediction as in regression likelihood. The model needs to be defined accordingly: -The forward pass during training takes x.shape == (batch_size, 2, dim) with -y.shape = (batch_size,). Meanwhile, during evaluation x.shape == (batch_size, dim). -Note that 'reward_modeling' only supports KronLaplace and DiagLaplace.
-
sigma_noise : torch.Tensor or float, default=1
-
observation noise for the regression setting; must be 1 for classification
-
prior_precision : torch.Tensor or float, default=1
-
prior precision of a Gaussian prior (= weight decay); -can be scalar, per-layer, or diagonal in the most general case
-
prior_mean : torch.Tensor or float, default=0
-
prior mean of a Gaussian prior, useful for continual learning
-
temperature : float, default=1
-
temperature of the likelihood; lower temperature leads to more -concentrated posterior and vice versa.
-
enable_backprop : bool, default=False
-
whether to enable backprop to the input x through the Laplace predictive. -Useful for e.g. Bayesian optimization.
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
backend : subclasses of CurvatureInterface
-
backend for access to curvature/Hessian approximations. Defaults to CurvlinopsGGN if None.
-
backend_kwargs : dict, default=None
-
arguments passed to the backend on initialization, for example to -set the number of MC samples for stochastic approximations.
-
asdl_fisher_kwargs : dict, default=None
-
arguments passed to the ASDL backend specifically on initialization.
-
-

Subclasses

- -

Instance variables

-
-
prop backend : CurvatureInterface
-
-
-
-
prop log_likelihood : torch.Tensor
-
-

Compute log likelihood on the training data after .fit() has been called. -The log likelihood is computed on-demand based on the loss and, for example, -the observation noise which makes it differentiable in the latter for -iterative updates.

-

Returns

-
-
log_likelihood : torch.Tensor
-
 
-
-
-
prop prior_precision_diag : torch.Tensor
-
-

Obtain the diagonal prior precision p_0 constructed from either -a scalar, layer-wise, or diagonal prior precision.

-

Returns

-
-
prior_precision_diag : torch.Tensor
-
 
-
-
-
prop prior_mean : torch.Tensor
-
-
-
-
prop prior_precision : torch.Tensor
-
-
-
-
prop sigma_noise : torch.Tensor
-
-
-
-
-

Methods

-
-
-def fit(self, train_loader: DataLoader) ‑> None -
-
-
-
-
-def log_marginal_likelihood(self, prior_precision: torch.Tensor | None = None, sigma_noise: torch.Tensor | None = None) -
-
-
-
-
-def predictive(self, x: torch.Tensor, pred_type: PredType | str, link_approx: LinkApprox | str, n_samples: int) -
-
-
-
-
-def optimize_prior_precision(self, pred_type: PredType | str, method: TuningMethod | str = TuningMethod.MARGLIK, n_steps: int = 100, lr: float = 0.1, init_prior_prec: float | torch.Tensor = 1.0, prior_structure: PriorStructure | str = PriorStructure.DIAG, val_loader: DataLoader | None = None, loss: torchmetrics.Metric | Callable[[torch.Tensor], torch.Tensor | float] | None = None, log_prior_prec_min: float = -4, log_prior_prec_max: float = 4, grid_size: int = 100, link_approx: LinkApprox | str = LinkApprox.PROBIT, n_samples: int = 100, verbose: bool = False, progress_bar: bool = False) -
-
-

Optimize the prior precision post-hoc using the method -specified by the user.

-

Parameters

-
-
pred_type : PredType or str in {'glm', 'nn'}
-
type of posterior predictive, linearized GLM predictive or neural -network sampling predictiv. The GLM predictive is consistent with the -curvature approximations used here.
-
method : TuningMethod or str in {'marglik', 'gridsearch'}, default=PredType.MARGLIK
-
specifies how the prior precision should be optimized.
-
n_steps : int, default=100
-
the number of gradient descent steps to take.
-
lr : float, default=1e-1
-
the learning rate to use for gradient descent.
-
init_prior_prec : float or tensor, default=1.0
-
initial prior precision before the first optimization step.
-
prior_structure : PriorStructure or str in {'scalar', 'layerwise', 'diag'}, default=PriorStructure.SCALAR
-
if init_prior_prec is scalar, the prior precision is optimized with this structure. -otherwise, the structure of init_prior_prec is maintained.
-
val_loader : torch.data.utils.DataLoader, default=None
-
DataLoader for the validation set; each iterate is a training batch (X, y).
-
loss : callable or torchmetrics.Metric, default=None
-
loss function to use for CV. If callable, the loss is computed offline (memory intensive). -If torchmetrics.Metric, running loss is computed (efficient). The default -depends on the likelihood: RunningNLLMetric() for classification and -reward modeling, running MeanSquaredError() for regression.
-
log_prior_prec_min : float, default=-4
-
lower bound of gridsearch interval.
-
log_prior_prec_max : float, default=4
-
upper bound of gridsearch interval.
-
grid_size : int, default=100
-
number of values to consider inside the gridsearch interval.
-
link_approx : LinkApprox or str in {'mc', 'probit', 'bridge'}, default=LinkApprox.PROBIT
-
how to approximate the classification link function for the 'glm'. -For pred_type='nn', only 'mc' is possible.
-
n_samples : int, default=100
-
number of samples for link_approx='mc'.
-
verbose : bool, default=False
-
if true, the optimized prior precision will be printed -(can be a large tensor if the prior has a diagonal covariance).
-
progress_bar : bool, default=False
-
whether to show a progress bar; updated at every batch-Hessian computation. -Useful for very large model and large amount of data, esp. when subset_of_weights='all'.
-
-
-
-
-
-class ParametricLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Parametric Laplace class.

-

Subclasses need to specify how the Hessian approximation is initialized, -how to add up curvature over training data, how to sample from the -Laplace approximation, and how to compute the functional variance.

-

A Laplace approximation is represented by a MAP which is given by the -model parameter and a posterior precision or covariance specifying -a Gaussian distribution \mathcal{N}(\theta_{MAP}, P^{-1}). -The goal of this class is to compute the posterior precision P -which sums as - -P = \sum_{n=1}^N \nabla^2_\theta \log p(\mathcal{D}_n \mid \theta) -\vert_{\theta_{MAP}} + \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}}. - -Every subclass implements different approximations to the log likelihood Hessians, -for example, a diagonal one. The prior is assumed to be Gaussian and therefore we have -a simple form for \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}} = P_0 . -In particular, we assume a scalar, layer-wise, or diagonal prior precision so that in -all cases P_0 = \textrm{diag}(p_0) and the structure of p_0 can be varied.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop scatter : torch.Tensor
-
-

Computes the scatter, a term of the log marginal likelihood that -corresponds to L-2 regularization: -scatter = (\theta_{MAP} - \mu_0)^{T} P_0 (\theta_{MAP} - \mu_0) .

-

Returns

-
-
scatter : torch.Tensor
-
 
-
-
-
prop log_det_prior_precision : torch.Tensor
-
-

Compute log determinant of the prior precision -\log \det P_0

-

Returns

-
-
log_det : torch.Tensor
-
 
-
-
-
prop log_det_posterior_precision : torch.Tensor
-
-

Compute log determinant of the posterior precision -\log \det P which depends on the subclasses structure -used for the Hessian approximation.

-

Returns

-
-
log_det : torch.Tensor
-
 
-
-
-
prop log_det_ratio : torch.Tensor
-
-

Compute the log determinant ratio, a part of the log marginal likelihood. - -\log \frac{\det P}{\det P_0} = \log \det P - \log \det P_0 -

-

Returns

-
-
log_det_ratio : torch.Tensor
-
 
-
-
-
prop posterior_precision : torch.Tensor
-
-

Compute or return the posterior precision P.

-

Returns

-
-
posterior_prec : torch.Tensor
-
 
-
-
-
-

Methods

-
-
-def fit(self, train_loader: DataLoader, override: bool = True, progress_bar: bool = False) ‑> None -
-
-

Fit the local Laplace approximation at the parameters of the model.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
each iterate is a training batch, either (X, y) tensors or a dict-like -object containing keys as expressed by self.dict_key_x and -self.dict_key_y. train_loader.dataset needs to be set to access -N, size of the data set.
-
override : bool, default=True
-
whether to initialize H, loss, and n_data again; setting to False is useful for -online learning settings to accumulate a sequential posterior approximation.
-
progress_bar : bool, default=False
-
whether to show a progress bar; updated at every batch-Hessian computation. -Useful for very large model and large amount of data, esp. when subset_of_weights='all'.
-
-
-
-def square_norm(self, value) ‑> torch.Tensor -
-
-

Compute the square norm under post. Precision with value-self.mean as 𝛥: - -\Delta^ -op P \Delta - -Returns

-
-
-
square_form
-
 
-
-
-
-def log_prob(self, value: torch.Tensor, normalized: bool = True) ‑> torch.Tensor -
-
-

Compute the log probability under the (current) Laplace approximation.

-

Parameters

-
-
value : torch.Tensor
-
 
-
normalized : bool, default=True
-
whether to return log of a properly normalized Gaussian or just the -terms that depend on value.
-
-

Returns

-
-
log_prob : torch.Tensor
-
 
-
-
-
-def log_marginal_likelihood(self, prior_precision: torch.Tensor | None = None, sigma_noise: torch.Tensor | None = None) -
-
-

Compute the Laplace approximation to the log marginal likelihood subject -to specific Hessian approximations that subclasses implement. -Requires that the Laplace approximation has been fit before. -The resulting torch.Tensor is differentiable in prior_precision and -sigma_noise if these have gradients enabled. -By passing prior_precision or sigma_noise, the current value is -overwritten. This is useful for iterating on the log marginal likelihood.

-

Parameters

-
-
prior_precision : torch.Tensor, optional
-
prior precision if should be changed from current prior_precision value
-
sigma_noise : torch.Tensor, optional
-
observation noise standard deviation if should be changed
-
-

Returns

-
-
log_marglik : torch.Tensor
-
 
-
-
-
-def predictive_samples(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], pred_type: PredType | str = PredType.GLM, n_samples: int = 100, diagonal_output: bool = False, generator: torch.Generator | None = None) -
-
-

Sample from the posterior predictive on input data x. -Can be used, for example, for Thompson sampling.

-

Parameters

-
-
x : torch.Tensor or MutableMapping
-
input data (batch_size, input_shape)
-
pred_type : {'glm', 'nn'}, default='glm'
-
type of posterior predictive, linearized GLM predictive or neural -network sampling predictive. The GLM predictive is consistent with -the curvature approximations used here.
-
n_samples : int
-
number of samples
-
diagonal_output : bool
-
whether to use a diagonalized glm posterior predictive on the outputs. -Only applies when pred_type='glm'.
-
generator : torch.Generator, optional
-
random number generator to control the samples (if sampling used)
-
-

Returns

-
-
samples : torch.Tensor
-
samples (n_samples, batch_size, output_shape)
-
-
-
-def functional_variance(self, Js: torch.Tensor) ‑> torch.Tensor -
-
-

Compute functional variance for the 'glm' predictive: -f_var[i] = Js[i] @ P.inv() @ Js[i].T, which is a output x output -predictive covariance matrix. -Mathematically, we have for a single Jacobian -\mathcal{J} = \nabla_\theta f(x;\theta)\vert_{\theta_{MAP}} -the output covariance matrix - \mathcal{J} P^{-1} \mathcal{J}^T .

-

Parameters

-
-
Js : torch.Tensor
-
Jacobians of model output wrt parameters -(batch, outputs, parameters)
-
-

Returns

-
-
f_var : torch.Tensor
-
output covariance (batch, outputs, outputs)
-
-
-
-def functional_covariance(self, Js: torch.Tensor) ‑> torch.Tensor -
-
-

Compute functional covariance for the 'glm' predictive: -f_cov = Js @ P.inv() @ Js.T, which is a batchoutput x batchoutput -predictive covariance matrix.

-

This emulates the GP posterior covariance N([f(x1), …,f(xm)], Cov[f(x1), …, f(xm)]). -Useful for joint predictions, such as in batched Bayesian optimization.

-

Parameters

-
-
Js : torch.Tensor
-
Jacobians of model output wrt parameters -(batch*outputs, parameters)
-
-

Returns

-
-
f_cov : torch.Tensor
-
output covariance (batch*outputs, batch*outputs)
-
-
-
-def sample(self, n_samples: int = 100, generator: torch.Generator | None = None) -
-
-

Sample from the Laplace posterior approximation, i.e., - \theta \sim \mathcal{N}(\theta_{MAP}, P^{-1}).

-

Parameters

-
-
n_samples : int, default=100
-
number of samples
-
generator : torch.Generator, optional
-
random number generator to control the samples
-
-

Returns

-
-
samples : torch.Tensor
-
 
-
-
-
-def state_dict(self) ‑> dict[str, typing.Any] -
-
-
-
-
-def load_state_dict(self, state_dict: dict[str, Any]) ‑> None -
-
-
-
-
-

Inherited members

- -
-
-class FunctionalLaplace -(model: nn.Module, likelihood: Likelihood | str, n_subset: int, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x='input_ids', dict_key_y='labels', backend: type[CurvatureInterface] | None = laplace.curvature.backpack.BackPackGGN, backend_kwargs: dict[str, Any] | None = None, independent_outputs: bool = False, seed: int = 0) -
-
-

Applying the GGN (Generalized Gauss-Newton) approximation for the Hessian in the Laplace approximation of the posterior -turns the underlying probabilistic model from a BNN into a GLM (generalized linear model). -This GLM (in the weight space) is equivalent to a GP (in the function space), see -Approximate Inference Turns Deep Networks into Gaussian Processes (Khan et al., 2019)

-

This class implements the (approximate) GP inference through which -we obtain the desired quantities (posterior predictive, marginal log-likelihood). -See Improving predictions of Bayesian neural nets via local linearization (Immer et al., 2021) -for more details.

-

Note that for likelihood='classification', we approximate L_{NN} with a diagonal matrix -( L_{NN} is a block-diagonal matrix, where blocks represent Hessians of per-data-point log-likelihood w.r.t. -neural network output f , See Appendix A.2.1 for exact definition). We -resort to such an approximation because of the (possible) errors found in Laplace approximation for -multiclass GP classification in Chapter 3.5 of R&W 2006 GP book, -see the question -here -for more details. Alternatively, one could also resort to one-vs-one or one-vs-rest implementations -for multiclass classification, however, that is not (yet) supported here.

-

Parameters

-
-
num_data : int
-
number of data points for Subset-of-Data (SOD) approximate GP inference.
-
diagonal_kernel : bool
-
GP kernel here is product of Jacobians, which results in a C \times C matrix where C is the output -dimension. If diagonal_kernel=True, only a diagonal of a GP kernel is used. This is (somewhat) equivalent to -assuming independent GPs across output channels.
-
-

See BaseLaplace class for the full interface.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop gp_kernel_prior_variance
-
-
-
-
prop log_det_ratio : torch.Tensor
-
-

Computes log determinant term in GP marginal likelihood

-

For classification we use eq. (3.44) from Chapter 3.5 from -GP book R&W 2006 with -(note that we always use diagonal approximation D of the Hessian of log likelihood w.r.t. f):

-

log determinant term := \log | I + D^{1/2}K D^{1/2} |

-

For regression, we use "standard" GP marginal likelihood:

-

log determinant term := \log | K + \sigma_2 I |

-
-
prop scatter : torch.Tensor
-
-

Compute scatter term in GP log marginal likelihood.

-

For classification we use eq. (3.44) from Chapter 3.5 from -GP book R&W 2006 with \hat{f} = f :

-

scatter term := f K^{-1} f^{T}

-

For regression, we use "standard" GP marginal likelihood:

-

scatter term := (y - m)K^{-1}(y -m )^T , -where m is the mean of the GP prior, which in our case corresponds to - m := f + J (\theta - \theta_{MAP})

-
-
prop prior_precision
-
-
-
-
-

Methods

-
-
-def fit(self, train_loader: DataLoader | MutableMapping, progress_bar: bool = False) -
-
-

Fit the Laplace approximation of a GP posterior.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
train_loader.dataset needs to be set to access N, size of the data set -train_loader.batch_size needs to be set to access b batch_size
-
progress_bar : bool
-
whether to show a progress bar during the fitting process.
-
-
-
-def predictive_samples(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], pred_type: PredType | str = PredType.GLM, n_samples: int = 100, diagonal_output: bool = False, generator: torch.Generator | None = None) -
-
-

Sample from the posterior predictive on input data x. -Can be used, for example, for Thompson sampling.

-

Parameters

-
-
x : torch.Tensor or MutableMapping
-
input data (batch_size, input_shape)
-
pred_type : {'glm'}, default='glm'
-
type of posterior predictive, linearized GLM predictive.
-
n_samples : int
-
number of samples
-
diagonal_output : bool
-
whether to use a diagonalized glm posterior predictive on the outputs. -Only applies when pred_type='glm'.
-
generator : torch.Generator, optional
-
random number generator to control the samples (if sampling used)
-
-

Returns

-
-
samples : torch.Tensor
-
samples (n_samples, batch_size, output_shape)
-
-
-
-def functional_variance(self, Js_star: torch.Tensor) ‑> torch.Tensor -
-
-

GP posterior variance:

-

k_{**} - K_{*M} (K_{MM}+ L_{MM}^{-1})^{-1} K_{M*}

-

Parameters

-
-
Js_star : torch.Tensor of shape (N*, C, P)
-
Jacobians of test data points
-
-

Returns

-
-
f_var : torch.Tensor of shape (N*,C, C)
-
Contains the posterior variances of N* testing points.
-
-
-
-def functional_covariance(self, Js_star: torch.Tensor) ‑> torch.Tensor -
-
-

GP posterior covariance:

-

k_{**} - K_{*M} (K_{MM}+ L_{MM}^{-1})^{-1} K_{M*}

-

Parameters

-
-
Js_star : torch.Tensor of shape (N*, C, P)
-
Jacobians of test data points
-
-

Returns

-
-
f_var : torch.Tensor of shape (N*xC, N*xC)
-
Contains the posterior covariances of N* testing points.
-
-
-
-def optimize_prior_precision(self, pred_type: PredType | str = PredType.GP, method: TuningMethod | str = TuningMethod.MARGLIK, n_steps: int = 100, lr: float = 0.1, init_prior_prec: float | torch.Tensor = 1.0, prior_structure: PriorStructure | str = PriorStructure.SCALAR, val_loader: DataLoader | None = None, loss: torchmetrics.Metric | Callable[[torch.Tensor], torch.Tensor | float] | None = None, log_prior_prec_min: float = -4, log_prior_prec_max: float = 4, grid_size: int = 100, link_approx: LinkApprox | str = LinkApprox.PROBIT, n_samples: int = 100, verbose: bool = False, progress_bar: bool = False) -
-
-

optimize_prior_precision_base from BaseLaplace with pred_type='gp'

-
-
-def log_marginal_likelihood(self, prior_precision: torch.Tensor | None = None, sigma_noise: torch.Tensor | None = None) -
-
-

Compute the Laplace approximation to the log marginal likelihood. -Requires that the Laplace approximation has been fit before. -The resulting torch.Tensor is differentiable in prior_precision and -sigma_noise if these have gradients enabled. -By passing prior_precision or sigma_noise, the current value is -overwritten. This is useful for iterating on the log marginal likelihood.

-

Parameters

-
-
prior_precision : torch.Tensor, optional
-
prior precision if should be changed from current prior_precision value
-
sigma_noise : torch.Tensor, optional
-
observation noise standard deviation if should be changed
-
-

Returns

-
-
log_marglik : torch.Tensor
-
 
-
-
-
-def state_dict(self) ‑> dict -
-
-
-
-
-def load_state_dict(self, state_dict: dict) -
-
-
-
-
-

Inherited members

- -
-
-class FullLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, backend_kwargs: dict[str, Any] | None = None) -
-
-

Laplace approximation with full, i.e., dense, log likelihood Hessian approximation -and hence posterior precision. Based on the chosen backend parameter, the full -approximation can be, for example, a generalized Gauss-Newton matrix. -Mathematically, we have P \in \mathbb{R}^{P \times P}. -See BaseLaplace for the full interface.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop posterior_scale : torch.Tensor
-
-

Posterior scale (square root of the covariance), i.e., -P^{-\frac{1}{2}}.

-

Returns

-
-
scale : torch.tensor
-
(parameters, parameters)
-
-
-
prop posterior_covariance : torch.Tensor
-
-

Posterior covariance, i.e., P^{-1}.

-

Returns

-
-
covariance : torch.tensor
-
(parameters, parameters)
-
-
-
prop posterior_precision : torch.Tensor
-
-

Posterior precision P.

-

Returns

-
-
precision : torch.tensor
-
(parameters, parameters)
-
-
-
-

Inherited members

- -
-
-class KronLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, damping: bool = False, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Laplace approximation with Kronecker factored log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have for each parameter group, e.g., torch.nn.Module, -that \P\approx Q \otimes H. -See BaseLaplace for the full interface and see -Kron and KronDecomposed for the structure of -the Kronecker factors. Kron is used to aggregate factors by summing up and -KronDecomposed is used to add the prior, a Hessian factor (e.g. temperature), -and computing posterior covariances, marginal likelihood, etc. -Damping can be enabled by setting damping=True.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop posterior_precision : KronDecomposed
-
-

Kronecker factored Posterior precision P.

-

Returns

-
-
precision : KronDecomposed
-
 
-
-
-
prop prior_precision : torch.Tensor
-
-
-
-
-

Methods

-
-
-def state_dict(self) ‑> dict[str, typing.Any] -
-
-
-
-
-def load_state_dict(self, state_dict: dict[str, Any]) -
-
-
-
-
-

Inherited members

- -
-
-class DiagLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Laplace approximation with diagonal log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have P \approx \textrm{diag}(P). -See BaseLaplace for the full interface.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop posterior_precision : torch.Tensor
-
-

Diagonal posterior precision p.

-

Returns

-
-
precision : torch.tensor
-
(parameters)
-
-
-
prop posterior_scale : torch.Tensor
-
-

Diagonal posterior scale \sqrt{p^{-1}}.

-

Returns

-
-
precision : torch.tensor
-
(parameters)
-
-
-
prop posterior_variance : torch.Tensor
-
-

Diagonal posterior variance p^{-1}.

-

Returns

-
-
precision : torch.tensor
-
(parameters)
-
-
-
-

Inherited members

- -
-
-class LowRankLaplace -(model: nn.Module, likelihood: Likelihood | str, backend: type[CurvatureInterface] = laplace.curvature.curvature.CurvatureInterface, sigma_noise: float | torch.Tensor = 1, prior_precision: float | torch.Tensor = 1, prior_mean: float | torch.Tensor = 0, temperature: float = 1, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend_kwargs: dict[str, Any] | None = None) -
-
-

Laplace approximation with low-rank log likelihood Hessian (approximation). -The low-rank matrix is represented by an eigendecomposition (vecs, values). -Based on the chosen backend, either a true Hessian or, for example, GGN -approximation could be used. -The posterior precision is computed as - P = V diag(l) V^T + P_0. -To sample, compute the functional variance, and log determinant, algebraic tricks -are usedto reduce the costs of inversion to the that of a K -imes K matrix -if we have a rank of K.

-

Note that only AsdfghjklHessian backend is supported. Install it via: -pip install git+https://git@github.com/wiseodd/asdl@asdfghjkl

-

See BaseLaplace for the full interface.

-

Ancestors

- -

Instance variables

-
-
prop V : torch.Tensor
-
-
-
-
prop Kinv : torch.Tensor
-
-
-
-
prop posterior_precision : tuple[tuple[torch.Tensor, torch.Tensor], torch.Tensor]
-
-

Return correctly scaled posterior precision that would be constructed -as H[0] @ diag(H[1]) @ H[0].T + self.prior_precision_diag.

-

Returns

-
-
H : tuple(eigenvectors, eigenvalues)
-
scaled self.H with temperature and loss factors.
-
prior_precision_diag : torch.Tensor
-
diagonal prior precision shape parameters to be added to H.
-
-
-
-

Inherited members

- -
-
-
-
- -
- - - diff --git a/examples/calibration_example.md b/docs/calibration_example.md similarity index 95% rename from examples/calibration_example.md rename to docs/calibration_example.md index 3d56c605..68c54733 100644 --- a/examples/calibration_example.md +++ b/docs/calibration_example.md @@ -1,8 +1,6 @@ -## Full example: _post-hoc_ Laplace on a large image classifier - An advantage of the Laplace approximation over variational Bayes and Markov Chain Monte Carlo methods is its _post-hoc_ nature. That means we can apply LA on (almost) any _pre-trained_ neural network. In this example, we will see how we can apply the last-layer LA on a deep WideResNet model, trained on CIFAR-10. -#### Data loading +## Data loading First, let us load the CIFAR-10 dataset. The helper scripts for CIFAR-10 and WideResNet are available in the `examples/helper` directory in the main repository. @@ -28,7 +26,7 @@ test_loader = dl.CIFAR10(train=False) targets = torch.cat([y for x, y in test_loader], dim=0).numpy() ``` -#### Load a pre-trained model +## Load a pre-trained model Next, we will load a pre-trained WideResNet-16-4 model. Note that a GPU with CUDA support is needed for this example. @@ -57,7 +55,7 @@ def predict(dataloader, model, laplace=False): return torch.cat(py).cpu().numpy() ``` -#### The calibration of MAP +## The calibration of MAP We are now ready to see how calibrated is the model. The metrics we use are the expected calibration error (ECE, Naeni et al., AAAI 2015) and the negative (Categorical) log-likelihood. Note that lower values are better for both these metrics. @@ -78,7 +76,7 @@ Running this snippet, we would get: [MAP] Acc.: 94.8%; ECE: 2.0%; NLL: 0.172 ``` -### The calibration of Laplace +## The calibration of Laplace Now we inspect the benefit of the LA. Let us apply the simple last-layer LA model, and optimize the prior precision hyperparameter using a _post-hoc_ marginal likelihood maximization. diff --git a/examples/calibration_gp_example.md b/docs/calibration_gp_example.md similarity index 92% rename from examples/calibration_gp_example.md rename to docs/calibration_gp_example.md index bf269fb8..fd17065b 100644 --- a/examples/calibration_gp_example.md +++ b/docs/calibration_gp_example.md @@ -1,13 +1,12 @@ -## Full example: Functional Laplace (GP) on FMNIST image classifier Applying the General-Gauss-Newton (GGN) approximation to the Hessian in the Laplace approximation (LA) of the BNN posterior turns the underlying probabilistic model from a BNN into a generalized linear model (GLM). -This GLM is equivalent to a Gaussian Process (GP) with a particular kernel [1, 2]. +This GLM is equivalent to a Gaussian Process (GP) with a particular kernel [1, 2]. -In this notebook, we will show how to use `laplace` library to perform GP inference on top of a *pre-trained* neural network. +In this notebook, we will show how to use `laplace` library to perform GP inference on top of a _pre-trained_ neural network. Note that a GPU with CUDA support is needed for this example. We recommend using a GPU with at least 24 GB of memory. If less memory is available, we suggest reducing `BATCH_SIZE` below. -#### Data loading +## Data loading First, let us load the FMIST dataset. The helper scripts for FMNIST and pre-trained CNN are available in the `examples/helper` directory in the main repository. @@ -36,11 +35,11 @@ test_loader = DataLoader(ds_test, batch_size=BATCH_SIZE, shuffle=False) targets = torch.cat([y for x, y in test_loader], dim=0).cpu() ``` -#### Load a pre-trained model +## Load a pre-trained model Next, we load a pre-trained CNN model. The code to train the model can be found in [BNN-predictions repo](https://github.com/AlexImmer/BNN-predictions). -``` python +```python MODEL_NAME = 'FMNIST_CNN_10_2.2e+02.pt' model = CIFAR10Net(ds_train.channels, ds_train.K, use_tanh=True).to('cuda') state = torch.load(f'helper/models/{MODEL_NAME}') @@ -51,7 +50,7 @@ prior_precision = state['delta'] To simplify the downstream tasks, we will use the following helper function to make predictions. It simply iterates through all minibatches and obtains the predictive probabilities of the FMNIST classes. -``` python +```python @torch.no_grad() def predict(dataloader, model, laplace=False): py = [] @@ -65,13 +64,13 @@ def predict(dataloader, model, laplace=False): return torch.cat(py).cpu().numpy() ``` -#### The calibration of MAP +## The calibration of MAP We are now ready to see how calibrated is the model. The metrics we use are the expected calibration error (ECE, Naeni et al., AAAI 2015) and the negative (Categorical) log-likelihood. Note that lower values are better for both these metrics. First, let us inspect the MAP model. We shall use the [`netcal`](https://github.com/fabiankueppers/calibration-framework) library to easily compute the ECE. -``` python +```python probs_map = predict(test_loader, model, laplace=False) acc_map = (probs_map.argmax(-1) == targets).float().mean() ece_map = ECE(bins=15).measure(probs_map.numpy(), targets.numpy()) @@ -86,19 +85,19 @@ Running this snippet, we would get: [MAP] Acc.: 91.7%; ECE: 1.6%; NLL: 0.253 ``` -### The calibration of Laplace +## The calibration of Laplace -Next, we run Laplace-GP inference to calibrate neural network's predictions. Since running exact GP inference is computationally infeasible, we perform Subset-of-Datapoints (SoD) [3] approximation here. In the code below, `m`denotes the number of datapoints used in the SoD posterior. +Next, we run Laplace-GP inference to calibrate neural network's predictions. Since running exact GP inference is computationally infeasible, we perform Subset-of-Datapoints (SoD) [3] approximation here. In the code below, `m`denotes the number of datapoints used in the SoD posterior. Execution of the cell below can take up to 5min (depending on the exact hardware used). -``` python +```python for m in [50, 200, 800, 1600]: print(f'Fitting Laplace-GP for m={m}') la = Laplace(model, 'classification', subset_of_weights='all', hessian_structure='gp', - diagonal_kernel=True, + diagonal_kernel=True, num_data=m, prior_precision=prior_precision) la.fit(train_loader) @@ -123,15 +122,11 @@ Fitting Laplace-GP for m=1600 ``` Notice that the post-hoc Laplace-GP inference does not have a significant impact on the accuracy, yet it improves the calibration (in terms of ECE) of the MAP model substantially. -
-
-
-
-### References +## References + [1] Khan, Mohammad Emtiyaz E., et al. "Approximate inference turns deep networks into gaussian processes." Advances in neural information processing systems 32 (2019) [2] Immer, Alexander, Maciej Korzepa, and Matthias Bauer. "Improving predictions of Bayesian neural nets via local linearization." International Conference on Artificial Intelligence and Statistics. PMLR, 2021 [3] Rasmussen, Carl Edward. "Gaussian processes in machine learning." Springer, 2004 - diff --git a/docs/curvature/asdfghjkl.html b/docs/curvature/asdfghjkl.html deleted file mode 100644 index 107f2d38..00000000 --- a/docs/curvature/asdfghjkl.html +++ /dev/null @@ -1,246 +0,0 @@ - - - - - - -laplace.curvature.asdfghjkl API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.curvature.asdfghjkl

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class AsdfghjklInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for asdfghjkl backend.

-

Ancestors

- -

Subclasses

- -

Methods

-
-
-def jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_\theta f(x;\theta) at current parameter \theta -using asdfghjkl's gradient per output dimension.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
enable_backprop : bool, default = False
-
whether to enable backprop through the Js and f w.r.t. x
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, parameters, outputs)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-def gradients(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor) -
-
-

Compute gradients \nabla_\theta \ell(f(x;\theta, y) at current parameter -\theta using asdfghjkl's backend.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
y : torch.Tensor
-
 
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
Gs : torch.Tensor
-
gradients (batch, parameters)
-
-
-
-

Inherited members

- -
-
-class AsdfghjklHessian -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', low_rank: int = 10) -
-
-

Interface for asdfghjkl backend.

-

Ancestors

- -

Methods

-
-
-def eig_lowrank(self, data_loader: DataLoader) ‑> tuple[torch.Tensor, torch.Tensor, torch.Tensor] -
-
-
-
-
-

Inherited members

- -
-
-class AsdfghjklGGN -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', stochastic: bool = False) -
-
-

Implementation of the GGNInterface using asdfghjkl.

-

Ancestors

- -

Inherited members

- -
-
-class AsdfghjklEF -(model: nn.Module, likelihood: Likelihood | None, last_layer: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Implementation of the EFInterface using asdfghjkl.

-

Ancestors

- -

Inherited members

- -
-
-
-
- -
- - - diff --git a/docs/curvature/asdl.html b/docs/curvature/asdl.html deleted file mode 100644 index 3c8611ac..00000000 --- a/docs/curvature/asdl.html +++ /dev/null @@ -1,243 +0,0 @@ - - - - - - -laplace.curvature.asdl API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.curvature.asdl

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class AsdlInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for asdfghjkl backend.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop loss_type : str
-
-
-
-
-

Methods

-
-
-def jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_\theta f(x;\theta) at current parameter \theta -using asdfghjkl's gradient per output dimension.

-

Parameters

-
-
x : torch.Tensor or MutableMapping (e.g. dict, UserDict)
-
input data (batch, input_shape) on compatible device with model if torch.Tensor. -If MutableMapping, then at least contains self.dict_key_x. -The latter is specific for reward modeling.
-
enable_backprop : bool, default = False
-
whether to enable backprop through the Js and f w.r.t. x
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, parameters, outputs)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-def gradients(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor) -
-
-

Compute gradients \nabla_\theta \ell(f(x;\theta, y) at current parameter -\theta using asdfghjkl's backend.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
y : torch.Tensor
-
 
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
Gs : torch.Tensor
-
gradients (batch, parameters)
-
-
-
-

Inherited members

- -
-
-class AsdlHessian -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for asdfghjkl backend.

-

Ancestors

- -

Inherited members

- -
-
-class AsdlGGN -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', stochastic: bool = False) -
-
-

Implementation of the GGNInterface using asdfghjkl.

-

Ancestors

- -

Inherited members

- -
-
-class AsdlEF -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Implementation of the EFInterface using asdfghjkl.

-

Ancestors

- -

Inherited members

- -
-
-
-
- -
- - - diff --git a/docs/curvature/backpack.html b/docs/curvature/backpack.html deleted file mode 100644 index 2c1bafb8..00000000 --- a/docs/curvature/backpack.html +++ /dev/null @@ -1,206 +0,0 @@ - - - - - - -laplace.curvature.backpack API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.curvature.backpack

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class BackPackInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for Backpack backend.

-

Ancestors

- -

Subclasses

- -

Methods

-
-
-def jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_{\theta} f(x;\theta) at current parameter \theta -using backpack's BatchGrad per output dimension. Note that BackPACK doesn't play well -with torch.func, so this method has to be overridden.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
enable_backprop : bool, default = False
-
whether to enable backprop through the Js and f w.r.t. x
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, parameters, outputs)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-def gradients(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor) -
-
-

Compute gradients \nabla_\theta \ell(f(x;\theta, y) at current parameter -\theta using Backpack's BatchGrad. Note that BackPACK doesn't play well -with torch.func, so this method has to be overridden.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
y : torch.Tensor
-
 
-
-

Returns

-
-
Gs : torch.Tensor
-
gradients (batch, parameters)
-
loss : torch.Tensor
-
 
-
-
-
-

Inherited members

- -
-
-class BackPackGGN -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', stochastic: bool = False) -
-
-

Implementation of the GGNInterface using Backpack.

-

Ancestors

- -

Inherited members

- -
-
-class BackPackEF -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Implementation of EFInterface using Backpack.

-

Ancestors

- -

Inherited members

- -
-
-
-
- -
- - - diff --git a/docs/curvature/curvature.html b/docs/curvature/curvature.html deleted file mode 100644 index 81539633..00000000 --- a/docs/curvature/curvature.html +++ /dev/null @@ -1,459 +0,0 @@ - - - - - - -laplace.curvature.curvature API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.curvature.curvature

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class CurvatureInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface to access curvature for a model and corresponding likelihood. -A CurvatureInterface must inherit from this baseclass and implement the -necessary functions jacobians, full, kron, and diag. -The interface might be extended in the future to account for other curvature -structures, for example, a block-diagonal one.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
torch model (neural network)
-
likelihood : {'classification', 'regression'}
-
 
-
last_layer : bool, default=False
-
only consider curvature of last layer
-
subnetwork_indices : torch.LongTensor, default=None
-
indices of the vectorized model parameters that define the subnetwork -to apply the Laplace approximation over
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
-

Attributes

-
-
lossfunc : torch.nn.MSELoss or torch.nn.CrossEntropyLoss
-
 
-
factor : float
-
conversion factor between torch losses and base likelihoods -For example, \frac{1}{2} to get to \mathcal{N}(f, 1) from MSELoss.
-
-

Subclasses

- -

Methods

-
-
-def jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_{\theta} f(x;\theta) at current parameter \theta, -via torch.func.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
enable_backprop : bool, default = False
-
whether to enable backprop through the Js and f w.r.t. x
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, parameters, outputs)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-def last_layer_jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_{\theta_\textrm{last}} f(x;\theta_\textrm{last}) -only at current last-layer parameter \theta_{\textrm{last}}.

-

Parameters

-
-
x : torch.Tensor
-
 
-
enable_backprop : bool, default=False
-
 
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, outputs, last-layer-parameters)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-def gradients(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor) -
-
-

Compute batch gradients \nabla_\theta \ell(f(x;\theta, y) at -current parameter \theta.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
y : torch.Tensor
-
 
-
-

Returns

-
-
Gs : torch.Tensor
-
gradients (batch, parameters)
-
loss : torch.Tensor
-
 
-
-
-
-def full(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, **kwargs: dict[str, Any]) -
-
-

Compute a dense curvature (approximation) in the form of a P \times P matrix -H with respect to parameters \theta \in \mathbb{R}^P.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H : torch.Tensor
-
Hessian approximation (parameters, parameters)
-
-
-
-def kron(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, N: int, **kwargs: dict[str, Any]) -
-
-

Compute a Kronecker factored curvature approximation (such as KFAC). -The approximation to H takes the form of two Kronecker factors Q, H, -i.e., H \approx Q \otimes H for each Module in the neural network permitting -such curvature. -Q is quadratic in the input-dimension of a module p_{in} \times p_{in} -and H in the output-dimension p_{out} \times p_{out}.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
N : int
-
total number of data points
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H : Kron
-
Kronecker factored Hessian approximation.
-
-
-
-def diag(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, **kwargs: dict[str, Any]) -
-
-

Compute a diagonal Hessian approximation to H and is represented as a -vector of the dimensionality of parameters \theta.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H : torch.Tensor
-
vector representing the diagonal of H
-
-
-
-def functorch_jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_{\theta} f(x;\theta) at current parameter \theta, -via torch.func.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
enable_backprop : bool, default = False
-
whether to enable backprop through the Js and f w.r.t. x
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, parameters, outputs)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-
-
-class GGNInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', stochastic: bool = False, num_samples: int = 1) -
-
-

Generalized Gauss-Newton or Fisher Curvature Interface. -The GGN is equal to the Fisher information for the available likelihoods. -In addition to CurvatureInterface, methods for Jacobians are required by subclasses.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
torch model (neural network)
-
likelihood : {'classification', 'regression'}
-
 
-
last_layer : bool, default=False
-
only consider curvature of last layer
-
subnetwork_indices : torch.Tensor, default=None
-
indices of the vectorized model parameters that define the subnetwork -to apply the Laplace approximation over
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
stochastic : bool, default=False
-
Fisher if stochastic else GGN
-
num_samples : int, default=1
-
Number of samples used to approximate the stochastic Fisher
-
-

Ancestors

- -

Subclasses

- -

Methods

-
-
-def full(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, **kwargs: dict[str, Any]) -
-
-

Compute the full GGN P \times P matrix as Hessian approximation -H_{ggn} with respect to parameters \theta \in \mathbb{R}^P. -For last-layer, reduced to \theta_{last}

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H : torch.Tensor
-
GGN (parameters, parameters)
-
-
-
-

Inherited members

- -
-
-class EFInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for Empirical Fisher as Hessian approximation. -In addition to CurvatureInterface, methods for gradients are required by subclasses.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
torch model (neural network)
-
likelihood : {'classification', 'regression'}
-
 
-
last_layer : bool, default=False
-
only consider curvature of last layer
-
subnetwork_indices : torch.Tensor, default=None
-
indices of the vectorized model parameters that define the subnetwork -to apply the Laplace approximation over
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
-

Attributes

-
-
lossfunc : torch.nn.MSELoss or torch.nn.CrossEntropyLoss
-
 
-
factor : float
-
conversion factor between torch losses and base likelihoods -For example, \frac{1}{2} to get to \mathcal{N}(f, 1) from MSELoss.
-
-

Ancestors

- -

Subclasses

- -

Methods

-
-
-def full(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, **kwargs: dict[str, Any]) -
-
-

Compute the full EF P \times P matrix as Hessian approximation -H_{ef} with respect to parameters \theta \in \mathbb{R}^P. -For last-layer, reduced to \theta_{last}

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H_ef : torch.Tensor
-
EF (parameters, parameters)
-
-
-
-

Inherited members

- -
-
-
-
- -
- - - diff --git a/docs/curvature/curvlinops.html b/docs/curvature/curvlinops.html deleted file mode 100644 index 86d331e1..00000000 --- a/docs/curvature/curvlinops.html +++ /dev/null @@ -1,187 +0,0 @@ - - - - - - -laplace.curvature.curvlinops API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.curvature.curvlinops

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class CurvlinopsInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for Curvlinops backend. https://github.com/f-dangel/curvlinops

-

Ancestors

- -

Subclasses

- -

Inherited members

- -
-
-class CurvlinopsGGN -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', stochastic: bool = False) -
-
-

Implementation of the GGNInterface using Curvlinops.

-

Ancestors

- -

Inherited members

- -
-
-class CurvlinopsEF -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Implementation of EFInterface using Curvlinops.

-

Ancestors

- -

Inherited members

- -
-
-class CurvlinopsHessian -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Implementation of the full Hessian using Curvlinops.

-

Ancestors

- -

Inherited members

- -
-
-
-
- -
- - - diff --git a/docs/curvature/index.html b/docs/curvature/index.html deleted file mode 100644 index bd65f715..00000000 --- a/docs/curvature/index.html +++ /dev/null @@ -1,935 +0,0 @@ - - - - - - -laplace.curvature API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.curvature

-
-
-
-
-

Sub-modules

-
-
laplace.curvature.asdfghjkl
-
-
-
-
laplace.curvature.asdl
-
-
-
-
laplace.curvature.backpack
-
-
-
-
laplace.curvature.curvature
-
-
-
-
laplace.curvature.curvlinops
-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class CurvatureInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface to access curvature for a model and corresponding likelihood. -A CurvatureInterface must inherit from this baseclass and implement the -necessary functions jacobians, full, kron, and diag. -The interface might be extended in the future to account for other curvature -structures, for example, a block-diagonal one.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
torch model (neural network)
-
likelihood : {'classification', 'regression'}
-
 
-
last_layer : bool, default=False
-
only consider curvature of last layer
-
subnetwork_indices : torch.LongTensor, default=None
-
indices of the vectorized model parameters that define the subnetwork -to apply the Laplace approximation over
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
-

Attributes

-
-
lossfunc : torch.nn.MSELoss or torch.nn.CrossEntropyLoss
-
 
-
factor : float
-
conversion factor between torch losses and base likelihoods -For example, \frac{1}{2} to get to \mathcal{N}(f, 1) from MSELoss.
-
-

Subclasses

- -

Methods

-
-
-def jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_{\theta} f(x;\theta) at current parameter \theta, -via torch.func.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
enable_backprop : bool, default = False
-
whether to enable backprop through the Js and f w.r.t. x
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, parameters, outputs)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-def last_layer_jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_{\theta_\textrm{last}} f(x;\theta_\textrm{last}) -only at current last-layer parameter \theta_{\textrm{last}}.

-

Parameters

-
-
x : torch.Tensor
-
 
-
enable_backprop : bool, default=False
-
 
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, outputs, last-layer-parameters)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-def gradients(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor) -
-
-

Compute batch gradients \nabla_\theta \ell(f(x;\theta, y) at -current parameter \theta.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
y : torch.Tensor
-
 
-
-

Returns

-
-
Gs : torch.Tensor
-
gradients (batch, parameters)
-
loss : torch.Tensor
-
 
-
-
-
-def full(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, **kwargs: dict[str, Any]) -
-
-

Compute a dense curvature (approximation) in the form of a P \times P matrix -H with respect to parameters \theta \in \mathbb{R}^P.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H : torch.Tensor
-
Hessian approximation (parameters, parameters)
-
-
-
-def kron(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, N: int, **kwargs: dict[str, Any]) -
-
-

Compute a Kronecker factored curvature approximation (such as KFAC). -The approximation to H takes the form of two Kronecker factors Q, H, -i.e., H \approx Q \otimes H for each Module in the neural network permitting -such curvature. -Q is quadratic in the input-dimension of a module p_{in} \times p_{in} -and H in the output-dimension p_{out} \times p_{out}.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
N : int
-
total number of data points
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H : Kron
-
Kronecker factored Hessian approximation.
-
-
-
-def diag(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, **kwargs: dict[str, Any]) -
-
-

Compute a diagonal Hessian approximation to H and is represented as a -vector of the dimensionality of parameters \theta.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H : torch.Tensor
-
vector representing the diagonal of H
-
-
-
-def functorch_jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_{\theta} f(x;\theta) at current parameter \theta, -via torch.func.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
enable_backprop : bool, default = False
-
whether to enable backprop through the Js and f w.r.t. x
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, parameters, outputs)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-
-
-class GGNInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', stochastic: bool = False, num_samples: int = 1) -
-
-

Generalized Gauss-Newton or Fisher Curvature Interface. -The GGN is equal to the Fisher information for the available likelihoods. -In addition to CurvatureInterface, methods for Jacobians are required by subclasses.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
torch model (neural network)
-
likelihood : {'classification', 'regression'}
-
 
-
last_layer : bool, default=False
-
only consider curvature of last layer
-
subnetwork_indices : torch.Tensor, default=None
-
indices of the vectorized model parameters that define the subnetwork -to apply the Laplace approximation over
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
stochastic : bool, default=False
-
Fisher if stochastic else GGN
-
num_samples : int, default=1
-
Number of samples used to approximate the stochastic Fisher
-
-

Ancestors

- -

Subclasses

- -

Methods

-
-
-def full(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, **kwargs: dict[str, Any]) -
-
-

Compute the full GGN P \times P matrix as Hessian approximation -H_{ggn} with respect to parameters \theta \in \mathbb{R}^P. -For last-layer, reduced to \theta_{last}

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H : torch.Tensor
-
GGN (parameters, parameters)
-
-
-
-

Inherited members

- -
-
-class EFInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for Empirical Fisher as Hessian approximation. -In addition to CurvatureInterface, methods for gradients are required by subclasses.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
torch model (neural network)
-
likelihood : {'classification', 'regression'}
-
 
-
last_layer : bool, default=False
-
only consider curvature of last layer
-
subnetwork_indices : torch.Tensor, default=None
-
indices of the vectorized model parameters that define the subnetwork -to apply the Laplace approximation over
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
-

Attributes

-
-
lossfunc : torch.nn.MSELoss or torch.nn.CrossEntropyLoss
-
 
-
factor : float
-
conversion factor between torch losses and base likelihoods -For example, \frac{1}{2} to get to \mathcal{N}(f, 1) from MSELoss.
-
-

Ancestors

- -

Subclasses

- -

Methods

-
-
-def full(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor, **kwargs: dict[str, Any]) -
-
-

Compute the full EF P \times P matrix as Hessian approximation -H_{ef} with respect to parameters \theta \in \mathbb{R}^P. -For last-layer, reduced to \theta_{last}

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape)
-
y : torch.Tensor
-
labels (batch, label_shape)
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
H_ef : torch.Tensor
-
EF (parameters, parameters)
-
-
-
-

Inherited members

- -
-
-class BackPackInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for Backpack backend.

-

Ancestors

- -

Subclasses

- -

Methods

-
-
-def jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_{\theta} f(x;\theta) at current parameter \theta -using backpack's BatchGrad per output dimension. Note that BackPACK doesn't play well -with torch.func, so this method has to be overridden.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
enable_backprop : bool, default = False
-
whether to enable backprop through the Js and f w.r.t. x
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, parameters, outputs)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-def gradients(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor) -
-
-

Compute gradients \nabla_\theta \ell(f(x;\theta, y) at current parameter -\theta using Backpack's BatchGrad. Note that BackPACK doesn't play well -with torch.func, so this method has to be overridden.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
y : torch.Tensor
-
 
-
-

Returns

-
-
Gs : torch.Tensor
-
gradients (batch, parameters)
-
loss : torch.Tensor
-
 
-
-
-
-

Inherited members

- -
-
-class BackPackGGN -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', stochastic: bool = False) -
-
-

Implementation of the GGNInterface using Backpack.

-

Ancestors

- -

Inherited members

- -
-
-class BackPackEF -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Implementation of EFInterface using Backpack.

-

Ancestors

- -

Inherited members

- -
-
-class AsdlInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for asdfghjkl backend.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop loss_type : str
-
-
-
-
-

Methods

-
-
-def jacobians(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], enable_backprop: bool = False) -
-
-

Compute Jacobians \nabla_\theta f(x;\theta) at current parameter \theta -using asdfghjkl's gradient per output dimension.

-

Parameters

-
-
x : torch.Tensor or MutableMapping (e.g. dict, UserDict)
-
input data (batch, input_shape) on compatible device with model if torch.Tensor. -If MutableMapping, then at least contains self.dict_key_x. -The latter is specific for reward modeling.
-
enable_backprop : bool, default = False
-
whether to enable backprop through the Js and f w.r.t. x
-
-

Returns

-
-
Js : torch.Tensor
-
Jacobians (batch, parameters, outputs)
-
f : torch.Tensor
-
output function (batch, outputs)
-
-
-
-def gradients(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], y: torch.Tensor) -
-
-

Compute gradients \nabla_\theta \ell(f(x;\theta, y) at current parameter -\theta using asdfghjkl's backend.

-

Parameters

-
-
x : torch.Tensor
-
input data (batch, input_shape) on compatible device with model.
-
y : torch.Tensor
-
 
-
-

Returns

-
-
loss : torch.Tensor
-
 
-
Gs : torch.Tensor
-
gradients (batch, parameters)
-
-
-
-

Inherited members

- -
-
-class AsdlGGN -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', stochastic: bool = False) -
-
-

Implementation of the GGNInterface using asdfghjkl.

-

Ancestors

- -

Inherited members

- -
-
-class AsdlEF -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Implementation of the EFInterface using asdfghjkl.

-

Ancestors

- -

Inherited members

- -
-
-class AsdlHessian -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for asdfghjkl backend.

-

Ancestors

- -

Inherited members

- -
-
-class CurvlinopsInterface -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Interface for Curvlinops backend. https://github.com/f-dangel/curvlinops

-

Ancestors

- -

Subclasses

- -

Inherited members

- -
-
-class CurvlinopsGGN -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', stochastic: bool = False) -
-
-

Implementation of the GGNInterface using Curvlinops.

-

Ancestors

- -

Inherited members

- -
-
-class CurvlinopsEF -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Implementation of EFInterface using Curvlinops.

-

Ancestors

- -

Inherited members

- -
-
-class CurvlinopsHessian -(model: nn.Module, likelihood: Likelihood | str, last_layer: bool = False, subnetwork_indices: torch.LongTensor | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Implementation of the full Hessian using Curvlinops.

-

Ancestors

- -

Inherited members

- -
-
-
-
- -
- - - diff --git a/examples/huggingface_example.md b/docs/huggingface_example.md similarity index 98% rename from examples/huggingface_example.md rename to docs/huggingface_example.md index 37df9f3b..8fec5278 100644 --- a/examples/huggingface_example.md +++ b/docs/huggingface_example.md @@ -1,5 +1,3 @@ -## Full Example: Applying Laplace on a Huggingface LLM model - In this example, we will see how to apply Laplace on a GPT2 Huggingface (HF) model. Laplace only has lightweight requirements for this; namely that the model's `forward` method must only take a single dict-like object (`dict`, `UserDict`, or in general, @@ -83,7 +81,7 @@ attention_mask torch.Size([4, 9]) labels torch.Size([4]) ``` -### Laplace on a subset of an LLM's weights +## Laplace on a subset of an LLM's weights Now, let's do the main "meat" of this example: Wrapping the HF model into a model that is compatible with Laplace. Notice that this wrapper just wraps the HF model and nothing else. @@ -248,7 +246,7 @@ As a final note, the dict-like input requirement of Laplace is very flexible. It be applicable to any tasks and any models. You just need to wrap the said model and make sure that your data loaders emit dict-like objects, where the input tensors are the dicts' values. -### Caveats +## Caveats Currently, diagonal EF with the Curvlinops backend is unsupported for dict-based inputs. This is because we use `torch.func`'s `vmap` to compute the diag-EF, and it only accepts diff --git a/docs/index.html b/docs/index.html deleted file mode 100644 index b9020716..00000000 --- a/docs/index.html +++ /dev/null @@ -1,3076 +0,0 @@ - - - - - - -laplace API documentation - - - - - - - - - - - - - -
-
-
-

Package laplace

-
-
-
-Laplace -![pytest](https://github.com/aleximmer/laplace/actions/workflows/pytest-default.yml/badge.svg) -![lint](https://github.com/aleximmer/laplace/actions/workflows/lint-ruff.yml/badge.svg) -![format](https://github.com/aleximmer/laplace/actions/workflows/format-ruff.yml/badge.svg) -
-

The laplace package facilitates the application of Laplace approximations for entire neural networks, subnetworks of neural networks, or just their last layer. -The package enables posterior approximations, marginal-likelihood estimation, and various posterior predictive computations. -The library documentation is available at https://aleximmer.github.io/Laplace.

-

There is also a corresponding paper, Laplace Redux — Effortless Bayesian Deep Learning, which introduces the library, provides an introduction to the Laplace approximation, reviews its use in deep learning, and empirically demonstrates its versatility and competitiveness. Please consider referring to the paper when using our library:

-
@inproceedings{laplace2021,
-  title={Laplace Redux--Effortless {B}ayesian Deep Learning},
-  author={Erik Daxberger and Agustinus Kristiadi and Alexander Immer
-          and Runa Eschenhagen and Matthias Bauer and Philipp Hennig},
-  booktitle={{N}eur{IPS}},
-  year={2021}
-}
-
-

The code to reproduce the experiments in the paper is also publicly available; it provides examples of how to use our library for predictive uncertainty quantification, model selection, and continual learning.

-
-

[!IMPORTANT] -As a user, one should not expect Laplace to work automatically. -That is, one should experiment with different Laplace's options -(hessian_factorization, prior precision tuning method, predictive method, backend, -etc!). Try looking at various papers that use Laplace for references on how to -set all those options depending on the applications/problems at hand.

-
-

Table of contents

-
    -
  1. Setup
    2. -
  2. Example usage
    3. -
  3. Simple usage
    4. -
  4. Marginal likelihood
    5. -
  5. Laplace on LLM
    6. -
  6. Subnetwork Laplace
    7. -
  7. Serialization
    8. -
  8. Structure
    9. -
  9. Extendability
    10. -
  10. When to use which backend?
    11. -
  11. Contributing
    12. -
  12. References
    13. -
-

Setup

-
-

[!IMPORTANT] -We assume Python >= 3.9 since lower versions are (soon to be) deprecated. -PyTorch version 2.0 and up is also required for full compatibility.

-
-

To install laplace with pip, run the following:

-
pip install laplace-torch
-
-

Additionally, if you want to use the asdfghjkl backend, please install it via:

-
pip install git+https://git@github.com/wiseodd/asdl@asdfghjkl
-
-

Setup dev environment

-

For development purposes, e.g. if you would like to make contributions, follow -the following steps:

-
    -
  1. Install uv
    2. -
  2. Then clone this repository and install the development dependencies:
    3. -
-
git clone git@github.com:aleximmer/Laplace.git
-uv sync --all-extras
-
-
    -
  1. laplace-torch is now available in editable mode, e.g. you can run:
    2. -
-
uv run python example/regression_example.py
-
-# Or, equivalently:
-source .venv/bin/activate
-python example/regression_example.py
-
-
-

[!NOTE] -See contributing guideline. -We're looking forward to your contributions!

-
-

Example usage

-

Simple usage

-

In the following example, a pre-trained model is loaded, -then the Laplace approximation is fit to the training data -(using a diagonal Hessian approximation over all parameters), -and the prior precision is optimized with cross-validation "gridsearch". -After that, the resulting LA is used for prediction with -the "probit" predictive for classification.

-
-

[!IMPORTANT] -Laplace expects all data loaders, e.g. train_loader and val_loader below, -to be instances of PyTorch -DataLoader. -Each batch, next(iter(data_loader)) must either be the standard (X, y) tensors -or a dict-like object containing at least the keys specified in -dict_key_x and dict_key_y in Laplace's constructor.

-

[!IMPORTANT] -The total number of data points in all data loaders must be accessible via -len(train_loader.dataset).

-

[!IMPORTANT] -In optimize_prior_precision, make sure to match the arguments with -the ones you want to pass in la(x, …) during prediction.

-
-
from laplace import Laplace
-
-# Pre-trained model
-model = load_map_model()
-
-# User-specified LA flavor
-la = Laplace(model, "classification",
-             subset_of_weights="all",
-             hessian_structure="diag")
-la.fit(train_loader)
-la.optimize_prior_precision(
-    method="gridsearch",
-    pred_type="glm",
-    link_approx="probit",
-    val_loader=val_loader
-)
-
-# User-specified predictive approx.
-pred = la(x, pred_type="glm", link_approx="probit")
-
-

Marginal likelihood

-

The marginal likelihood can be used for model selection [10] and is differentiable -for continuous hyperparameters like the prior precision or observation noise. -Here, we fit the library default, KFAC last-layer LA and differentiate -the log marginal likelihood.

-
from laplace import Laplace
-
-# Un- or pre-trained model
-model = load_model()
-
-# Default to recommended last-layer KFAC LA:
-la = Laplace(model, likelihood="regression")
-la.fit(train_loader)
-
-# ML w.r.t. prior precision and observation noise
-ml = la.log_marginal_likelihood(prior_prec, obs_noise)
-ml.backward()
-
-

Laplace on LLM

-
-

[!TIP] -This library also supports Huggingface models and parameter-efficient fine-tuning. -See examples/huggingface_examples.py and examples/huggingface_examples.md -for the full exposition.

-
-

First, we need to wrap the pretrained model so that the forward method takes a -dict-like input. Note that when you iterate over a Huggingface dataloader, -this is what you get by default. Having a dict-like input is nice since different models -have different number of inputs (e.g. GPT-like LLMs only take input_ids, while BERT-like -ones take both input_ids and attention_mask, etc.). Inside this forward method you -can do your usual preprocessing like moving the tensor inputs into the correct device.

-
class MyGPT2(nn.Module):
-    def __init__(self, tokenizer: PreTrainedTokenizer) -> None:
-        super().__init__()
-        config = GPT2Config.from_pretrained("gpt2")
-        config.pad_token_id = tokenizer.pad_token_id
-        config.num_labels = 2
-        self.hf_model = GPT2ForSequenceClassification.from_pretrained(
-            "gpt2", config=config
-        )
-
-    def forward(self, data: MutableMapping) -> torch.Tensor:
-        device = next(self.parameters()).device
-        input_ids = data["input_ids"].to(device)
-        attn_mask = data["attention_mask"].to(device)
-        output_dict = self.hf_model(input_ids=input_ids, attention_mask=attn_mask)
-        return output_dict.logits
-
-

Then you can "select" which parameters of the LLM you want to apply the Laplace approximation -on, by switching off the gradients of the "unneeded" parameters. -For example, we can replicate a last-layer Laplace: (in actual practice, use Laplace(..., subset_of_weights='last_layer', ...) instead, though!)

-
model = MyGPT2(tokenizer)
-model.eval()
-
-# Enable grad only for the last layer
-for p in model.hf_model.parameters():
-    p.requires_grad = False
-for p in model.hf_model.score.parameters():
-    p.requires_grad = True
-
-la = Laplace(
-    model,
-    likelihood="classification",
-    # Will only hit the last-layer since it's the only one that is grad-enabled
-    subset_of_weights="all",
-    hessian_structure="diag",
-)
-la.fit(dataloader)
-la.optimize_prior_precision()
-
-test_data = next(iter(dataloader))
-pred = la(test_data)
-
-

This is useful because we can apply the LA only on the parameter-efficient finetuning -weights. E.g., we can fix the LLM itself, and apply the Laplace approximation only -on the LoRA weights. Huggingface will automatically switch off the non-LoRA weights' -gradients.

-
def get_lora_model():
-    model = MyGPT2(tokenizer)  # Note we don't disable grad
-    config = LoraConfig(
-        r=4,
-        lora_alpha=16,
-        target_modules=["c_attn"],  # LoRA on the attention weights
-        lora_dropout=0.1,
-        bias="none",
-    )
-    lora_model = get_peft_model(model, config)
-    return lora_model
-
-lora_model = get_lora_model()
-
-# Train it as usual here...
-
-lora_model.eval()
-
-lora_la = Laplace(
-    lora_model,
-    likelihood="classification",
-    subset_of_weights="all",
-    hessian_structure="diag",
-    backend=AsdlGGN,
-)
-
-test_data = next(iter(dataloader))
-lora_pred = lora_la(test_data)
-
-

Subnetwork Laplace

-

This example shows how to fit the Laplace approximation over only -a subnetwork within a neural network (while keeping all other parameters -fixed at their MAP estimates), as proposed in [11]. It also exemplifies -different ways to specify the subnetwork to perform inference over.

-

First, we make use of SubnetLaplace, where we specify the subnetwork by -generating a list of indices for the active model parameters.

-
from laplace import Laplace
-
-# Pre-trained model
-model = load_model()
-
-# Examples of different ways to specify the subnetwork
-# via indices of the vectorized model parameters
-#
-# Example 1: select the 128 parameters with the largest magnitude
-from laplace.utils import LargestMagnitudeSubnetMask
-subnetwork_mask = LargestMagnitudeSubnetMask(model, n_params_subnet=128)
-subnetwork_indices = subnetwork_mask.select()
-
-# Example 2: specify the layers that define the subnetwork
-from laplace.utils import ModuleNameSubnetMask
-subnetwork_mask = ModuleNameSubnetMask(model, module_names=["layer.1", "layer.3"])
-subnetwork_mask.select()
-subnetwork_indices = subnetwork_mask.indices
-
-# Example 3: manually define the subnetwork via custom subnetwork indices
-import torch
-subnetwork_indices = torch.tensor([0, 4, 11, 42, 123, 2021])
-
-# Define and fit subnetwork LA using the specified subnetwork indices
-la = Laplace(model, "classification",
-             subset_of_weights="subnetwork",
-             hessian_structure="full",
-             subnetwork_indices=subnetwork_indices)
-la.fit(train_loader)
-
-

Besides SubnetLaplace, you can, as already mentioned, also treat the last -layer only using Laplace(..., subset_of_weights='last_layer'), which uses -LLLaplace. As a third method, you may define a subnetwork by disabling -gradients of fixed model parameters. The different methods target different use -cases. Each method has pros and cons, please see this -discussion -for details. In summary

-
    -
  • Disable-grad: General method to perform Laplace on specific types of -layer/parameter, e.g. in an LLM with LoRA. Can be used to emulate LLLaplace -as well. Always use subset_of_weights='all' for this method.
    • -
  • subnet selection by disabling grads is more efficient than -SubnetLaplace since it avoids calculating full Jacobians first
    • -
  • disabling grads can only be performed on Parameter level and not for -individual weights, so this doesn't cover all cases that SubnetLaplace -offers such as Largest*SubnetMask or RandomSubnetMask
    • -
  • LLLaplace: last-layer specific code with improved performance (#145)
    • -
  • SubnetLaplace: more fine-grained partitioning such as -LargestMagnitudeSubnetMask
    • -
-

Serialization

-

As with plain torch, we support to ways to serialize data.

-

One is the familiar state_dict approach. Here you need to save and re-create -both model and Laplace(). Use this for long-term storage of models and -sharing of a fitted Laplace() instance.

-
# Save model and Laplace instance
-torch.save(model.state_dict(), "model_state_dict.bin")
-torch.save(la.state_dict(), "la_state_dict.bin")
-
-# Load serialized data
-model2 = MyModel(...)
-model2.load_state_dict(torch.load("model_state_dict.bin"))
-la2 = Laplace(model2, "classification",
-              subset_of_weights="all",
-              hessian_structure="diag")
-la2.load_state_dict(torch.load("la_state_dict.bin"))
-
-

The second approach is to save the whole Laplace() object, including -self.model. This is less verbose and more convenient since you have the -trained model and the fitted Laplace() data stored in one place, but also comes with -some -drawbacks. -Use this for quick save-load cycles during experiments, say.

-
# Save Laplace, including la.model
-torch.save(la, "la.pt")
-
-# Load both
-torch.load("la.pt")
-
-

Some Laplace variants such as LLLaplace might have trouble being serialized -using the default pickle module, which torch.save() and torch.load() use -(AttributeError: Can't pickle local object ...). In this case, the -dill package will come in handy.

-
import dill
-
-torch.save(la, "la.pt", pickle_module=dill)
-
-

With both methods, you are free to switch devices, for instance when you -trained on a GPU but want to run predictions on CPU. In this case, use

-
torch.load(..., map_location="cpu")
-
-
-

[!WARNING] -Currently, this library always assumes that the model has an -output tensor of shape (batch_size, …, n_classes), so in -the case of image outputs, you need to rearrange from NCHW to NHWC.

-
-

Structure

-

The laplace package consists of two main components:

-
    -
  1. The subclasses of laplace.BaseLaplace that implement different sparsity structures: different subsets of weights ('all', 'subnetwork' and 'last_layer') and different structures of the Hessian approximation ('full', 'kron', 'lowrank', 'diag' and 'gp'). This results in ten currently available options: FullLaplace, KronLaplace, DiagLaplace, FunctionalLaplace the corresponding last-layer variations FullLLLaplace, KronLLLaplace, DiagLLLaplace and FunctionalLLLaplace (which are all subclasses of laplace.LLLaplace), laplace.SubnetLaplace (which only supports 'full' and 'diag' Hessian approximations) and LowRankLaplace (which only supports inference over 'all' weights). All of these can be conveniently accessed via the laplace.Laplace function.
    2. -
  2. The backends in laplace.curvature which provide access to Hessian approximations of -the corresponding sparsity structures, for example, the diagonal GGN.
    3. -
-

Additionally, the package provides utilities for -decomposing a neural network into feature extractor and last layer for LLLaplace subclasses (laplace.utils.feature_extractor) -and -effectively dealing with Kronecker factors (laplace.utils.matrix).

-

Finally, the package implements several options to select/specify a subnetwork for SubnetLaplace (as subclasses of laplace.utils.subnetmask.SubnetMask). -Automatic subnetwork selection strategies include: uniformly at random (RandomSubnetMask), by largest parameter magnitudes (LargestMagnitudeSubnetMask), and by largest marginal parameter variances (LargestVarianceDiagLaplaceSubnetMask and LargestVarianceSWAGSubnetMask). -In addition to that, subnetworks can also be specified manually, by listing the names of either the model parameters (ParamNameSubnetMask) or modules (ModuleNameSubnetMask) to perform Laplace inference over.

-

Extendability

-

To extend the laplace package, new BaseLaplace subclasses can be designed, for example, -Laplace with a block-diagonal Hessian structure. -One can also implement custom subnetwork selection strategies as new subclasses of SubnetMask.

-

Alternatively, extending or integrating backends (subclasses of curvature.curvature) allows to provide different Hessian -approximations to the Laplace approximations. -For example, currently the curvature.CurvlinopsInterface based on Curvlinops and the native torch.func (previously known as functorch), curvature.BackPackInterface based on BackPACK and curvature.AsdlInterface based on ASDL are available.

-

When to use which backend

-
-

[!TIP] -Each backend as its own caveat/behavior. The use the following to guide you -picking the suitable backend, depending on you model & application.

-
-
    -
  • Small, simple MLP, or last-layer Laplace: Any backend should work well. -CurvlinopsGGN or CurvlinopsEF is recommended if -hessian_factorization = 'kron', but it's inefficient for other factorizations.
    • -
  • LLMs with PEFT (e.g. LoRA): AsdlGGN and AsdlEF are recommended.
    • -
  • Continuous Bayesian optimization: CurvlinopsGGN/EF and BackpackGGN/EF are -recommended since they are the only ones supporting backprop over Jacobians.
    • -
-
-

[!CAUTION] -The curvlinops backends are inefficient for full and diagonal factorizations. -Moreover, they're also inefficient for computing the Jacobians of large models -since they rely on torch.func.jacrev along torch.func.vmap! -Finally, curvlinops only computes K-FAC (hessian_factorization = 'kron') -for nn.Linear and nn.Conv2d modules (including those inside larger modules -like Attention).

-

[!CAUTION] -The BackPack backends are limited to models expressed as nn.Sequential. -Also, they're not compatible with normalization layers.

-
-

Documentation

-

The documentation is available here or can be generated and/or viewed locally:

-
# assuming the repository was cloned
-uv sync --all-extras
-# create docs and write to html
-uv run bash update_docs.sh
-# .. or serve the docs directly
-uv run pdoc --http 0.0.0.0:8080 laplace --template-dir template
-
-

Contributing

-

Pull requests are very welcome. Please follow these guidelines:

-
    -
  1. Install Laplace via uv sync --all-extras which will install ruff and all requirements necessary to run the tests and build the docs.
    2. -
  2. Use ruff as autoformatter. Please refer to the following makefile and run it via make ruff. Please note that the order of ruff check --fix and ruff format is important!
    3. -
  3. Also use ruff as linter. Please manually fix all linting errors/warnings before opening a pull request.
    4. -
  4. Fully document your changes in the form of Python docstrings, typehinting, and (if applicable) code/markdown examples in the ./examples subdirectory.
    5. -
  5. Provide as many test cases as possible. Make sure all test cases pass.
    6. -
-

Issues, bug reports, and ideas are also very welcome!

- - -

References

-

This package relies on various improvements to the Laplace approximation for neural networks, which was originally due to MacKay [1]. Please consider citing the respective papers if you use any of their proposed methods via our laplace library.

- -

Full example: Optimization of the marginal likelihood and prediction

-

Sinusoidal toy data

-

We show how the marginal likelihood can be used after training a MAP network on a simple sinusoidal regression task. -Subsequently, we use the optimized LA to predict which provides uncertainty on top of the MAP prediction. -We also show how the marglik_training() utility method can be used to jointly train the MAP and hyperparameters. -First, we set up the training data for the problem with observation noise \(\sigma=0.3\):

-
from laplace.baselaplace import FullLaplace
-from laplace.curvature.backpack import BackPackGGN
-import numpy as np
-import torch
-
-from laplace import Laplace, marglik_training
-
-from helper.dataloaders import get_sinusoid_example
-from helper.util import plot_regression
-
-n_epochs = 1000
-torch.manual_seed(711)
-# sample toy data example
-X_train, y_train, train_loader, X_test = get_sinusoid_example(sigma_noise=0.3)
-
-

Training a MAP

-

We now use pytorch to train a neural network with single hidden layer and Tanh activation. -The trained neural network will be our MAP estimate. -This is standard so nothing new here, yet:

-
# create and train MAP model
-def get_model():
-    torch.manual_seed(711)
-    return torch.nn.Sequential(
-        torch.nn.Linear(1, 50), torch.nn.Tanh(), torch.nn.Linear(50, 1)
-    )
-model = get_model()
-
-criterion = torch.nn.MSELoss()
-optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)
-for i in range(n_epochs):
-    for X, y in train_loader:
-        optimizer.zero_grad()
-        loss = criterion(model(X), y)
-        loss.backward()
-        optimizer.step()
-
-

Fitting and optimizing the Laplace approximation using empirical Bayes

-

With the MAP-trained model at hand, we can estimate the prior precision and observation noise -using empirical Bayes after training. -The Laplace() method is called to construct a LA for "regression" with "all" weights. -As default Laplace() returns a Kronecker factored LA, we use "full" instead on this small example. -We fit the LA to the training data and initialize log_prior and log_sigma. -Using Adam, we minimize the negative log marginal likelihood for n_epochs.

-
la = Laplace(model, "regression", subset_of_weights="all", hessian_structure="full")
-la.fit(train_loader)
-log_prior, log_sigma = torch.ones(1, requires_grad=True), torch.ones(1, requires_grad=True)
-hyper_optimizer = torch.optim.Adam([log_prior, log_sigma], lr=1e-1)
-for i in range(n_epochs):
-    hyper_optimizer.zero_grad()
-    neg_marglik = - la.log_marginal_likelihood(log_prior.exp(), log_sigma.exp())
-    neg_marglik.backward()
-    hyper_optimizer.step()
-
-

The obtained observation noise is close to the ground truth with a value of \(\sigma \approx 0.28\) -without the need for any validation data. -The resulting prior precision is \(\delta \approx 0.10\).

-

Bayesian predictive

-

Here, we compare the MAP prediction to the obtained LA prediction. -For LA, we have a closed-form predictive distribution on the output \(f\) which is a Gaussian -\(\mathcal{N}(f(x;\theta_{MAP}), \mathbb{V}[f] + \sigma^2)\):

-
x = X_test.flatten().cpu().numpy()
-f_mu, f_var = la(X_test)
-f_mu = f_mu.squeeze().detach().cpu().numpy()
-f_sigma = f_var.squeeze().sqrt().cpu().numpy()
-pred_std = np.sqrt(f_sigma**2 + la.sigma_noise.item()**2)
-
-plot_regression(X_train, y_train, x, f_mu, pred_std)
-
-

-

:align: center

-

In comparison to the MAP, the predictive shows useful uncertainties. -When our MAP is over or underfit, the Laplace approximation cannot fix this anymore. -In this case, joint optimization of MAP and marginal likelihood can be useful.

-

Jointly optimize MAP and hyperparameters using online empirical Bayes

-

We provide a utility method marglik_training() that implements the algorithm proposed in [1]. -The method optimizes the neural network and the hyperparameters in an interleaved way -and returns an optimally regularized LA. -Below, we use this method and plot the corresponding predictive uncertainties again:

-
model = get_model()
-la, model, margliks, losses = marglik_training(
-    model=model, train_loader=train_loader, likelihood="regression",
-    hessian_structure="full", backend=BackPackGGN, n_epochs=n_epochs,
-    optimizer_kwargs={"lr": 1e-2}, prior_structure="scalar"
-)
-
-f_mu, f_var = la(X_test)
-f_mu = f_mu.squeeze().detach().cpu().numpy()
-f_sigma = f_var.squeeze().sqrt().cpu().numpy()
-pred_std = np.sqrt(f_sigma**2 + la.sigma_noise.item()**2)
-
-plot_regression(X_train, y_train, x, f_mu, pred_std)
-
-

-

:align: center

-

Full example: post-hoc Laplace on a large image classifier

-

An advantage of the Laplace approximation over variational Bayes and Markov Chain Monte Carlo methods is its post-hoc nature. That means we can apply LA on (almost) any pre-trained neural network. In this example, we will see how we can apply the last-layer LA on a deep WideResNet model, trained on CIFAR-10.

-

Data loading

-

First, let us load the CIFAR-10 dataset. The helper scripts for CIFAR-10 and WideResNet are available in the examples/helper directory in the main repository.

-
import torch
-import torch.distributions as dists
-import numpy as np
-import helper.wideresnet as wrn
-import helper.dataloaders as dl
-from helper import util
-from netcal.metrics import ECE
-
-from laplace import Laplace
-
-
-np.random.seed(7777)
-torch.manual_seed(7777)
-torch.backends.cudnn.deterministic = True
-torch.backends.cudnn.benchmark = True
-
-train_loader = dl.CIFAR10(train=True)
-test_loader = dl.CIFAR10(train=False)
-targets = torch.cat([y for x, y in test_loader], dim=0).numpy()
-
-

Load a pre-trained model

-

Next, we will load a pre-trained WideResNet-16-4 model. Note that a GPU with CUDA support is needed for this example.

-
# The model is a standard WideResNet 16-4
-# Taken as is from https://github.com/hendrycks/outlier-exposure
-model = wrn.WideResNet(16, 4, num_classes=10).cuda().eval()
-
-util.download_pretrained_model()
-model.load_state_dict(torch.load("./temp/CIFAR10_plain.pt"))
-
-

To simplify the downstream tasks, we will use the following helper function to make predictions. It simply iterates through all minibatches and obtains the predictive probabilities of the CIFAR-10 classes.

-
@torch.no_grad()
-def predict(dataloader, model, laplace=False):
-    py = []
-
-    for x, _ in dataloader:
-        if laplace:
-            py.append(model(x.cuda()))
-        else:
-            py.append(torch.softmax(model(x.cuda()), dim=-1))
-
-    return torch.cat(py).cpu().numpy()
-
-

The calibration of MAP

-

We are now ready to see how calibrated is the model. The metrics we use are the expected calibration error (ECE, Naeni et al., AAAI 2015) and the negative (Categorical) log-likelihood. Note that lower values are better for both these metrics.

-

First, let us inspect the MAP model. We shall use the netcal library to easily compute the ECE.

-
probs_map = predict(test_loader, model, laplace=False)
-acc_map = (probs_map.argmax(-1) == targets).float().mean()
-ece_map = ECE(bins=15).measure(probs_map.numpy(), targets.numpy())
-nll_map = -dists.Categorical(probs_map).log_prob(targets).mean()
-
-print(f"[MAP] Acc.: {acc_map:.1%}; ECE: {ece_map:.1%}; NLL: {nll_map:.3}")
-
-

Running this snippet, we would get:

-
[MAP] Acc.: 94.8%; ECE: 2.0%; NLL: 0.172
-
-

The calibration of Laplace

-

Now we inspect the benefit of the LA. Let us apply the simple last-layer LA model, and optimize the prior precision hyperparameter using a post-hoc marginal likelihood maximization.

-
# Laplace
-la = Laplace(model, "classification",
-             subset_of_weights="last_layer",
-             hessian_structure="kron")
-la.fit(train_loader)
-la.optimize_prior_precision(method="marglik")
-
-

Then, we are ready to see how well does LA improves the calibration of the MAP model:

-
probs_laplace = predict(test_loader, la, laplace=True)
-acc_laplace = (probs_laplace.argmax(-1) == targets).float().mean()
-ece_laplace = ECE(bins=15).measure(probs_laplace.numpy(), targets.numpy())
-nll_laplace = -dists.Categorical(probs_laplace).log_prob(targets).mean()
-
-print(f"[Laplace] Acc.: {acc_laplace:.1%}; ECE: {ece_laplace:.1%}; NLL: {nll_laplace:.3}")
-
-

Running this snippet, we obtain:

-
[Laplace] Acc.: 94.8%; ECE: 0.8%; NLL: 0.157
-
-

Notice that the last-layer LA does not do any harm to the accuracy, yet it improves the calibration of the MAP model substantially.

-

Full Example: Applying Laplace on a Huggingface LLM model

-

In this example, we will see how to apply Laplace on a GPT2 Huggingface (HF) model. -Laplace only has lightweight requirements for this; namely that the model's forward -method must only take a single dict-like object (dict, UserDict, or in general, -collections.abc.MutableMapping). This is entirely compatible with HF since HF's -data loaders are assumed to emit an object derived from UserDict. However, you -need to ensure this yourself — you need to wrap the standard HF model to conform -to that requirement. Also, you need to e.g. do torch.to(device) inside the -said forward method.

-

Let's start with as usual with importing stuff.

-
from collections.abc import MutableMapping
-from collections import UserDict
-import numpy
-import torch
-from torch import nn
-import torch.utils.data as data_utils
-
-from laplace import Laplace
-
-import logging
-import warnings
-
-logging.basicConfig(level="ERROR")
-warnings.filterwarnings("ignore")
-
-from transformers import ( # noqa: E402
-    GPT2Config,
-    GPT2ForSequenceClassification,
-    GPT2Tokenizer,
-    DataCollatorWithPadding,
-    PreTrainedTokenizer,
-)
-from peft import LoraConfig, get_peft_model # noqa: E402
-from datasets import Dataset # noqa: E402
-
-# make deterministic
-
-torch.manual_seed(0)
-numpy.random.seed(0)
-
-

Next, we create a toy dataset. You can use any HF datasets or your own, of course.

-
tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
-tokenizer.pad_token_id = tokenizer.eos_token_id
-
-data = [
-    {"text": "Today is hot, but I will manage!!!!", "label": 1},
-    {"text": "Tomorrow is cold", "label": 0},
-    {"text": "Carpe diem", "label": 1},
-    {"text": "Tempus fugit", "label": 1},
-]
-dataset = Dataset.from_list(data)
-
-def tokenize(row):
-    return tokenizer(row["text"])
-
-dataset = dataset.map(tokenize, remove_columns=["text"])
-dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "label"])
-dataloader = data_utils.DataLoader(
-    dataset, batch_size=100, collate_fn=DataCollatorWithPadding(tokenizer)
-)
-
-data = next(iter(dataloader))
-print(
-    f"Huggingface data defaults to UserDict, which is a MutableMapping? {isinstance(data, UserDict)}"
-)
-for k, v in data.items():
-    print(k, v.shape)
-
-

This is the output:

-
Huggingface data defaults to UserDict, which is a MutableMapping? True
-input_ids torch.Size([4, 9])
-attention_mask torch.Size([4, 9])
-labels torch.Size([4])
-
-

Laplace on a subset of an LLM's weights

-

Now, let's do the main "meat" of this example: Wrapping the HF model into a model that is -compatible with Laplace. Notice that this wrapper just wraps the HF model and nothing else. -Notice also we do inputs.to(device) inside self.forward().

-
class MyGPT2(nn.Module):
-    """
-    Huggingface LLM wrapper.
-
-    Args:
-        tokenizer: The tokenizer used for preprocessing the text data. Needed
-            since the model needs to know the padding token id.
-    """
-
-    def __init__(self, tokenizer: PreTrainedTokenizer) -> None:
-        super().__init__()
-        config = GPT2Config.from_pretrained("gpt2")
-        config.pad_token_id = tokenizer.pad_token_id
-        config.num_labels = 2
-        self.hf_model = GPT2ForSequenceClassification.from_pretrained(
-            "gpt2", config=config
-        )
-
-    def forward(self, data: MutableMapping) -> torch.Tensor:
-        """
-        Custom forward function. Handles things like moving the
-        input tensor to the correct device inside.
-
-        Args:
-            data: A dict-like data structure with `input_ids` inside.
-                This is the default data structure assumed by Huggingface
-                dataloaders.
-
-        Returns:
-            logits: An `(batch_size, n_classes)`-sized tensor of logits.
-        """
-        device = next(self.parameters()).device
-        input_ids = data["input_ids"].to(device)
-        attn_mask = data["attention_mask"].to(device)
-        output_dict = self.hf_model(input_ids=input_ids, attention_mask=attn_mask)
-        return output_dict.logits
-
-model = MyGPT2(tokenizer)
-
-

Now, let's apply Laplace. Let's do a last-layer Laplace first. -Notice that we add -an argument feature_reduction there. This is because Huggingface models reduce the -logits and not the features.

-
model = MyGPT2(tokenizer)
-model.eval()
-
-la = Laplace(
-    model,
-    likelihood="classification",
-    subset_of_weights="last_layer",
-    hessian_structure="full",
-    # This must reflect faithfully the reduction technique used in the model
-    # Otherwise, correctness is not guaranteed
-    feature_reduction="pick_last",
-)
-la.fit(dataloader)
-la.optimize_prior_precision()
-
-X_test = next(iter(dataloader))
-print(f"[Last-layer Laplace] The predictive tensor is of shape: {la(X_test).shape}.")
-
-

Here's the output:

-
[Last-layer Laplace] The predictive tensor is of shape: torch.Size([4, 2]).
-
-

Subnetwork Laplace

-

Also, we can do the same thing by switching off the gradients of all layers except the -top layer. Laplace will automatically only compute the Hessian (and Jacobians) of the -parameters in which requires_grad is True.

-

Notice that you can "mix-and-match" this gradient switching. You can do a subnetwork Laplace -easily by doing so!

-
model.eval()
-
-# Enable grad only for the last layer
-
-for p in model.hf_model.parameters():
-    p.requires_grad = False
-
-for p in model.hf_model.score.parameters():
-    p.requires_grad = True
-
-la = Laplace(
-    model,
-    # Will only hit the last-layer since it's the only one that is grad-enabled
-    likelihood="classification",
-    subset_of_weights="all",
-    hessian_structure="diag",
-)
-la.fit(dataloader)
-la.optimize_prior_precision()
-
-X_test = next(iter(dataloader))
-print(f"[Subnetwork Laplace] The predictive tensor is of shape: {la(X_test).shape}.")
-
-

Here are the outputs to validate that Laplace works:

-
[Subnetwork Laplace] The predictive tensor is of shape: torch.Size([4, 2]).
-
-

Full Laplace on LoRA parameters only

-

Of course, you can also apply Laplace on the parameter-efficient fine tuning weights (like LoRA). -To do this, simply extend your LLM with LoRA, using HF's peft library, and apply Laplace as -usual. Note that peft automatically switches off the non-LoRA weights.

-
def get_lora_model():
-    model = MyGPT2(tokenizer) # Note we don't disable grad
-    config = LoraConfig(
-        r=4,
-        lora_alpha=16,
-        target_modules=["c_attn"], # LoRA on the attention weights
-        lora_dropout=0.1,
-        bias="none",
-    )
-    lora_model = get_peft_model(model, config)
-    return lora_model
-
-lora_model = get_lora_model()
-
-# Train it as usual
-
-lora_model.eval()
-
-lora_la = Laplace(
-    lora_model,
-    likelihood="classification",
-    subset_of_weights="all",
-    hessian_structure="kron",
-)
-lora_la.fit(dataloader)
-
-X_test = next(iter(dataloader))
-print(f"[LoRA-LLM] The predictive tensor is of shape: {lora_la(X_test).shape}.")
-
-

Here is the output, as expected:

-
[LoRA-LLM] The predictive tensor is of shape: torch.Size([4, 2]).
-
-

As a final note, the dict-like input requirement of Laplace is very flexible. It can essentially -be applicable to any tasks and any models. You just need to wrap the said model and make sure -that your data loaders emit dict-like objects, where the input tensors are the dicts' values.

-

Caveats

-

Currently, diagonal EF with the Curvlinops backend is unsupported for dict-based inputs. -This is because we use torch.func's vmap to compute the diag-EF, and it only accepts -tensor input in the model's forward. -See this issue. -So, if you can write down your Huggingface model's forward to accept only a single tensor, -this is much preferable.

-

For instance, in the case of causal LLM like GPTs, only input_ids -tensor is necessary. -Then, any backend and any hessian factorization can be used in this case.

-

Otherwise, if you must use dict-based inputs, choose the following backends:

-
    -
  • CurvlinopsGGN for hessian_factorization = {"kron", "diag"}
    • -
  • CurvlinopsEF for hessian_factorization = {"kron"}
    • -
  • AsdlGGN for hessian_factorization = {"kron", "diag"}
    • -
  • AsdlEF for hessian_factorization = {"kron", "diag"}
    • -
-

Full Example: Bayesian Bradley-Terry Reward Modeling

-

The laplace-torch library can also be used to "Bayesianize" a pretrained Bradley-Terry -reward model, popular in large language models. See http://arxiv.org/abs/2009.01325 -for a primer in reward modeling.

-

First order of business, let's define our comparison dataset. We will use the datasets -library from Huggingface to handle the data.

-
import numpy as np
-import torch
-from torch import nn, optim
-from torch.nn import functional as F
-import torch.utils.data as data_utils
-
-from datasets import Dataset
-
-from laplace import Laplace
-
-import logging
-import warnings
-
-logging.basicConfig(level="ERROR")
-warnings.filterwarnings("ignore")
-
-# make deterministic
-torch.manual_seed(0)
-np.random.seed(0)
-
-
-# Pairwise comparison dataset. The label indicates which `x0` or `x1` is preferred.
-data_dict = [
-    {
-        "x0": torch.randn(3),
-        "x1": torch.randn(3),
-        "label": torch.randint(2, size=(1,)).item(),
-    }
-    for _ in range(10)
-]
-dataset = Dataset.from_list(data_dict)
-
-

Now, let's define the reward model. During training, it assumes that x is a tensor -of shape (batch_size, 2, dim), which is a concatenation of x0 and x1 above. -The second dimension of size 2 is preserved through the forward pass, resulting in -a logit tensor of shape (batch_size, 2) (the network itself is single-output). -Then, the standard cross-entropy loss is applied.

-

Note that this requirement is quite weak and can covers general cases. However, if you -prefer to use the dict-like inputs as in Huggingface LLM models, this can also be done. -Simply combine what you have learned from this example with the Huggingface LLM example -provided in this library.

-

During testing, this model behaves like a standard single-output regression -model.

-
class SimpleRewardModel(nn.Module):
-    """A simple reward model, compatible with the Bradley-Terry likelihood.
-    """
-
-    def __init__(self):
-        super().__init__()
-        self.net = nn.Sequential(nn.Linear(3, 100), nn.ReLU(), nn.Linear(100, 1))
-
-    def forward(self, x):
-        """Args:
-            x: torch.Tensor
-                If training == True then shape (batch_size, 2, dim)
-                Else shape (batch_size, dim)
-
-        Returns:
-            logits: torch.Tensor
-                If training then shape (batch_size, 2)
-                Else shape (batch_size, 1)
-        """
-        if len(x.shape) == 3:
-            batch_size, _, dim = x.shape
-
-            # Flatten to (batch_size*2, dim)
-            flat_x = x.reshape(-1, dim)
-
-            # Forward
-            flat_logits = self.net(flat_x)  # (batch_size*2, 1)
-
-            # Reshape back to (batch_size, 2)
-            return flat_logits.reshape(batch_size, 2)
-        else:
-            logits = self.net(x)  # (batch_size, 1)
-            return logits
-
-

To fulfill the 3D tensor requirement, we need to preprocess the dict-based dataset.

-
# Preprocess to coalesce x0 and x1 into a single array/tensor
-def append_x0_x1(row):
-    # The tensor values above are automatically casted as lists by `Dataset`
-    row["x"] = np.stack([row["x0"], row["x1"]])  # (2, dim)
-    return row
-
-
-tensor_dataset = dataset.map(append_x0_x1, remove_columns=["x0", "x1"])
-tensor_dataset.set_format(type="torch", columns=["x", "label"])
-tensor_dataloader = data_utils.DataLoader(
-    data_utils.TensorDataset(tensor_dataset["x"], tensor_dataset["label"]), batch_size=3
-)
-
-

Then, we can train as usual using the cross entropy loss.

-
reward_model = SimpleRewardModel()
-opt = optim.AdamW(reward_model.parameters(), weight_decay=1e-3)
-
-# Train as usual
-for epoch in range(10):
-    for x, y in tensor_dataloader:
-        opt.zero_grad()
-        out = reward_model(x)
-        loss = F.cross_entropy(out, y)
-        loss.backward()
-        opt.step()
-
-

Applying Laplace to this model is a breeze. Simply state that the likelihood is reward_modeling.

-
# Laplace !!! Notice the likelihood !!!
-reward_model.eval()
-la = Laplace(reward_model, likelihood="reward_modeling", subset_of_weights="all")
-la.fit(tensor_dataloader)
-la.optimize_prior_precision()
-
-

As we can see, during prediction, even though we train & fit Laplace using the cross entropy -loss (i.e. classification), in test time, the model behaves like a regression model. -So, you don't get probability vectors as outputs. Instead, you get two tensors -containing the predictive means and predictive variance.

-
x_test = torch.randn(5, 3)
-pred_mean, pred_var = la(x_test)
-print(
-    f"Input shape {tuple(x_test.shape)}, predictive mean of shape "
-    + f"{tuple(pred_mean.shape)}, predictive covariance of shape "
-    + f"{tuple(pred_var.shape)}"
-)
-
-

Here's the output:

-
Input shape (5, 3), predictive mean of shape (5, 1), predictive covariance of shape (5, 1, 1)
-
-
-
-

Sub-modules

-
-
laplace.baselaplace
-
-
-
-
laplace.curvature
-
-
-
-
laplace.laplace
-
-
-
-
laplace.lllaplace
-
-
-
-
laplace.subnetlaplace
-
-
-
-
laplace.utils
-
-
-
-
-
-
-
-
-

Functions

-
-
-def Laplace(model: torch.nn.Module, likelihood: Likelihood | str, subset_of_weights: SubsetOfWeights | str = SubsetOfWeights.LAST_LAYER, hessian_structure: HessianStructure | str = HessianStructure.KRON, *args, **kwargs) -
-
-

Simplified Laplace access using strings instead of different classes.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
likelihood : Likelihood or str in {'classification', 'regression'}
-
 
-
subset_of_weights : SubsetofWeights or {'last_layer', 'subnetwork', 'all'}, default=SubsetOfWeights.LAST_LAYER
-
subset of weights to consider for inference
-
hessian_structure : HessianStructure or str in {'diag', 'kron', 'full', 'lowrank', 'gp'}, default=HessianStructure.KRON
-
structure of the Hessian approximation (note that in case of 'gp', -we are not actually doing any Hessian approximation, the inference is instead done in the functional space)
-
-

Returns

-
-
laplace : BaseLaplace
-
chosen subclass of BaseLaplace instantiated with additional arguments
-
-
-
-def marglik_training(model: torch.nn.Module, train_loader: DataLoader, likelihood: Likelihood | str = Likelihood.CLASSIFICATION, hessian_structure: HessianStructure | str = HessianStructure.KRON, backend: Type[CurvatureInterface] = laplace.curvature.asdl.AsdlGGN, optimizer_cls: Type[Optimizer] = torch.optim.adam.Adam, optimizer_kwargs: dict | None = None, scheduler_cls: Type[LRScheduler] | None = None, scheduler_kwargs: dict | None = None, n_epochs: int = 300, lr_hyp: float = 0.1, prior_structure: PriorStructure | str = PriorStructure.LAYERWISE, n_epochs_burnin: int = 0, n_hypersteps: int = 10, marglik_frequency: int = 1, prior_prec_init: float = 1.0, sigma_noise_init: float = 1.0, temperature: float = 1.0, fix_sigma_noise: bool = False, progress_bar: bool = False, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels') -
-
-

Marginal-likelihood based training (Algorithm 1 in [1]). -Optimize model parameters and hyperparameters jointly. -Model parameters are optimized to minimize negative log joint (train loss) -while hyperparameters minimize negative log marginal likelihood.

-

This method replaces standard neural network training and adds hyperparameter -optimization to the procedure.

-

The settings of standard training can be controlled by passing train_loader, -optimizer_cls, optimizer_kwargs, scheduler_cls, scheduler_kwargs, and n_epochs. -The model should return logits, i.e., no softmax should be applied. -With likelihood=Likelihood.CLASSIFICATION or Likelihood.REGRESSION, one can choose between -categorical likelihood (CrossEntropyLoss) and Gaussian likelihood (MSELoss).

-

As in [1], we optimize prior precision and, for regression, observation noise -using the marginal likelihood. The prior precision structure can be chosen -as 'scalar', 'layerwise', or 'diagonal'. 'layerwise' is a good default -and available to all Laplace approximations. lr_hyp is the step size of the -Adam hyperparameter optimizer, n_hypersteps controls the number of steps -for each estimated marginal likelihood, n_epochs_burnin controls how many -epochs to skip marginal likelihood estimation, marglik_frequency controls -how often to estimate the marginal likelihood (default of 1 re-estimates -after every epoch, 5 would estimate every 5-th epoch).

-

References

-

[1] Immer, A., Bauer, M., Fortuin, V., Rätsch, G., Khan, EM. -Scalable Marginal Likelihood Estimation for Model Selection in Deep Learning. -ICML 2021.

-

Parameters

-
-
model : torch.nn.Module
-
torch neural network model (needs to comply with Backend choice)
-
train_loader : DataLoader
-
pytorch dataloader that implements len(train_loader.dataset) to obtain number of data points
-
likelihood : str, default=Likelihood.CLASSIFICATION
-
Likelihood.CLASSIFICATION or Likelihood.REGRESSION
-
hessian_structure : {'diag', 'kron', 'full'}, default='kron'
-
structure of the Hessian approximation
-
backend : Backend, default=AsdlGGN
-
Curvature subclass, e.g. AsdlGGN/AsdlEF or BackPackGGN/BackPackEF
-
optimizer_cls : torch.optim.Optimizer, default=Adam
-
optimizer to use for optimizing the neural network parameters togeth with train_loader
-
optimizer_kwargs : dict, default=None
-
keyword arguments for optimizer_cls, for example to change learning rate or momentum
-
scheduler_cls : torch.optim.lr_scheduler._LRScheduler, default=None
-
optionally, a scheduler to use on the learning rate of the optimizer. -scheduler.step() is called after every batch of the standard training.
-
scheduler_kwargs : dict, default=None
-
keyword arguments for scheduler_cls, e.g. lr_min for CosineAnnealingLR
-
n_epochs : int, default=300
-
number of epochs to train for
-
lr_hyp : float, default=0.1
-
Adam learning rate for hyperparameters
-
prior_structure : str, default='layerwise'
-
structure of the prior. one of ['scalar', 'layerwise', 'diag']
-
n_epochs_burnin : int default=0
-
how many epochs to train without estimating and differentiating marglik
-
n_hypersteps : int, default=10
-
how many steps to take on the hyperparameters when marglik is estimated
-
marglik_frequency : int
-
how often to estimate (and differentiate) the marginal likelihood -marglik_frequency=1 would be every epoch, -marglik_frequency=5 would be every 5 epochs.
-
prior_prec_init : float, default=1.0
-
initial prior precision
-
sigma_noise_init : float, default=1.0
-
initial observation noise (for regression only)
-
temperature : float, default=1.0
-
factor for the likelihood for 'overcounting' data. Might be required for data augmentation.
-
fix_sigma_noise : bool, default=False
-
if False, optimize observation noise via marglik otherwise use sigma_noise_init throughout. -Only works for regression.
-
progress_bar : bool, default=False
-
whether to show a progress bar (updated per epoch) or not
-
enable_backprop : bool, default=False
-
make the returned Laplace instance backpropable—useful for e.g. Bayesian optimization.
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
-

Returns

-
-
lap : laplace.laplace
-
fit Laplace approximation with the best obtained marginal likelihood during training
-
model : torch.nn.Module
-
corresponding model with the MAP parameters
-
margliks : list
-
list of marginal likelihoods obtained during training (to monitor convergence)
-
losses : list
-
list of losses (log joints) obtained during training (to monitor convergence)
-
-
-
-
-
-

Classes

-
-
-class BaseLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Baseclass for all Laplace approximations in this library.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
likelihood : Likelihood or str in {'classification', 'regression', 'reward_modeling'}
-
determines the log likelihood Hessian approximation. -In the case of 'reward_modeling', it fits Laplace using the classification likelihood, -then does prediction as in regression likelihood. The model needs to be defined accordingly: -The forward pass during training takes x.shape == (batch_size, 2, dim) with -y.shape = (batch_size,). Meanwhile, during evaluation x.shape == (batch_size, dim). -Note that 'reward_modeling' only supports KronLaplace and DiagLaplace.
-
sigma_noise : torch.Tensor or float, default=1
-
observation noise for the regression setting; must be 1 for classification
-
prior_precision : torch.Tensor or float, default=1
-
prior precision of a Gaussian prior (= weight decay); -can be scalar, per-layer, or diagonal in the most general case
-
prior_mean : torch.Tensor or float, default=0
-
prior mean of a Gaussian prior, useful for continual learning
-
temperature : float, default=1
-
temperature of the likelihood; lower temperature leads to more -concentrated posterior and vice versa.
-
enable_backprop : bool, default=False
-
whether to enable backprop to the input x through the Laplace predictive. -Useful for e.g. Bayesian optimization.
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
backend : subclasses of CurvatureInterface
-
backend for access to curvature/Hessian approximations. Defaults to CurvlinopsGGN if None.
-
backend_kwargs : dict, default=None
-
arguments passed to the backend on initialization, for example to -set the number of MC samples for stochastic approximations.
-
asdl_fisher_kwargs : dict, default=None
-
arguments passed to the ASDL backend specifically on initialization.
-
-

Subclasses

- -

Instance variables

-
-
prop backend : CurvatureInterface
-
-
-
-
prop log_likelihood : torch.Tensor
-
-

Compute log likelihood on the training data after .fit() has been called. -The log likelihood is computed on-demand based on the loss and, for example, -the observation noise which makes it differentiable in the latter for -iterative updates.

-

Returns

-
-
log_likelihood : torch.Tensor
-
 
-
-
-
prop prior_precision_diag : torch.Tensor
-
-

Obtain the diagonal prior precision p_0 constructed from either -a scalar, layer-wise, or diagonal prior precision.

-

Returns

-
-
prior_precision_diag : torch.Tensor
-
 
-
-
-
prop prior_mean : torch.Tensor
-
-
-
-
prop prior_precision : torch.Tensor
-
-
-
-
prop sigma_noise : torch.Tensor
-
-
-
-
-

Methods

-
-
-def fit(self, train_loader: DataLoader) ‑> None -
-
-
-
-
-def log_marginal_likelihood(self, prior_precision: torch.Tensor | None = None, sigma_noise: torch.Tensor | None = None) -
-
-
-
-
-def predictive(self, x: torch.Tensor, pred_type: PredType | str, link_approx: LinkApprox | str, n_samples: int) -
-
-
-
-
-def optimize_prior_precision(self, pred_type: PredType | str, method: TuningMethod | str = TuningMethod.MARGLIK, n_steps: int = 100, lr: float = 0.1, init_prior_prec: float | torch.Tensor = 1.0, prior_structure: PriorStructure | str = PriorStructure.DIAG, val_loader: DataLoader | None = None, loss: torchmetrics.Metric | Callable[[torch.Tensor], torch.Tensor | float] | None = None, log_prior_prec_min: float = -4, log_prior_prec_max: float = 4, grid_size: int = 100, link_approx: LinkApprox | str = LinkApprox.PROBIT, n_samples: int = 100, verbose: bool = False, progress_bar: bool = False) -
-
-

Optimize the prior precision post-hoc using the method -specified by the user.

-

Parameters

-
-
pred_type : PredType or str in {'glm', 'nn'}
-
type of posterior predictive, linearized GLM predictive or neural -network sampling predictiv. The GLM predictive is consistent with the -curvature approximations used here.
-
method : TuningMethod or str in {'marglik', 'gridsearch'}, default=PredType.MARGLIK
-
specifies how the prior precision should be optimized.
-
n_steps : int, default=100
-
the number of gradient descent steps to take.
-
lr : float, default=1e-1
-
the learning rate to use for gradient descent.
-
init_prior_prec : float or tensor, default=1.0
-
initial prior precision before the first optimization step.
-
prior_structure : PriorStructure or str in {'scalar', 'layerwise', 'diag'}, default=PriorStructure.SCALAR
-
if init_prior_prec is scalar, the prior precision is optimized with this structure. -otherwise, the structure of init_prior_prec is maintained.
-
val_loader : torch.data.utils.DataLoader, default=None
-
DataLoader for the validation set; each iterate is a training batch (X, y).
-
loss : callable or torchmetrics.Metric, default=None
-
loss function to use for CV. If callable, the loss is computed offline (memory intensive). -If torchmetrics.Metric, running loss is computed (efficient). The default -depends on the likelihood: RunningNLLMetric() for classification and -reward modeling, running MeanSquaredError() for regression.
-
log_prior_prec_min : float, default=-4
-
lower bound of gridsearch interval.
-
log_prior_prec_max : float, default=4
-
upper bound of gridsearch interval.
-
grid_size : int, default=100
-
number of values to consider inside the gridsearch interval.
-
link_approx : LinkApprox or str in {'mc', 'probit', 'bridge'}, default=LinkApprox.PROBIT
-
how to approximate the classification link function for the 'glm'. -For pred_type='nn', only 'mc' is possible.
-
n_samples : int, default=100
-
number of samples for link_approx='mc'.
-
verbose : bool, default=False
-
if true, the optimized prior precision will be printed -(can be a large tensor if the prior has a diagonal covariance).
-
progress_bar : bool, default=False
-
whether to show a progress bar; updated at every batch-Hessian computation. -Useful for very large model and large amount of data, esp. when subset_of_weights='all'.
-
-
-
-
-
-class ParametricLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Parametric Laplace class.

-

Subclasses need to specify how the Hessian approximation is initialized, -how to add up curvature over training data, how to sample from the -Laplace approximation, and how to compute the functional variance.

-

A Laplace approximation is represented by a MAP which is given by the -model parameter and a posterior precision or covariance specifying -a Gaussian distribution \mathcal{N}(\theta_{MAP}, P^{-1}). -The goal of this class is to compute the posterior precision P -which sums as - -P = \sum_{n=1}^N \nabla^2_\theta \log p(\mathcal{D}_n \mid \theta) -\vert_{\theta_{MAP}} + \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}}. - -Every subclass implements different approximations to the log likelihood Hessians, -for example, a diagonal one. The prior is assumed to be Gaussian and therefore we have -a simple form for \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}} = P_0 . -In particular, we assume a scalar, layer-wise, or diagonal prior precision so that in -all cases P_0 = \textrm{diag}(p_0) and the structure of p_0 can be varied.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop scatter : torch.Tensor
-
-

Computes the scatter, a term of the log marginal likelihood that -corresponds to L-2 regularization: -scatter = (\theta_{MAP} - \mu_0)^{T} P_0 (\theta_{MAP} - \mu_0) .

-

Returns

-
-
scatter : torch.Tensor
-
 
-
-
-
prop log_det_prior_precision : torch.Tensor
-
-

Compute log determinant of the prior precision -\log \det P_0

-

Returns

-
-
log_det : torch.Tensor
-
 
-
-
-
prop log_det_posterior_precision : torch.Tensor
-
-

Compute log determinant of the posterior precision -\log \det P which depends on the subclasses structure -used for the Hessian approximation.

-

Returns

-
-
log_det : torch.Tensor
-
 
-
-
-
prop log_det_ratio : torch.Tensor
-
-

Compute the log determinant ratio, a part of the log marginal likelihood. - -\log \frac{\det P}{\det P_0} = \log \det P - \log \det P_0 -

-

Returns

-
-
log_det_ratio : torch.Tensor
-
 
-
-
-
prop posterior_precision : torch.Tensor
-
-

Compute or return the posterior precision P.

-

Returns

-
-
posterior_prec : torch.Tensor
-
 
-
-
-
-

Methods

-
-
-def fit(self, train_loader: DataLoader, override: bool = True, progress_bar: bool = False) ‑> None -
-
-

Fit the local Laplace approximation at the parameters of the model.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
each iterate is a training batch, either (X, y) tensors or a dict-like -object containing keys as expressed by self.dict_key_x and -self.dict_key_y. train_loader.dataset needs to be set to access -N, size of the data set.
-
override : bool, default=True
-
whether to initialize H, loss, and n_data again; setting to False is useful for -online learning settings to accumulate a sequential posterior approximation.
-
progress_bar : bool, default=False
-
whether to show a progress bar; updated at every batch-Hessian computation. -Useful for very large model and large amount of data, esp. when subset_of_weights='all'.
-
-
-
-def square_norm(self, value) ‑> torch.Tensor -
-
-

Compute the square norm under post. Precision with value-self.mean as 𝛥: - -\Delta^ -op P \Delta - -Returns

-
-
-
square_form
-
 
-
-
-
-def log_prob(self, value: torch.Tensor, normalized: bool = True) ‑> torch.Tensor -
-
-

Compute the log probability under the (current) Laplace approximation.

-

Parameters

-
-
value : torch.Tensor
-
 
-
normalized : bool, default=True
-
whether to return log of a properly normalized Gaussian or just the -terms that depend on value.
-
-

Returns

-
-
log_prob : torch.Tensor
-
 
-
-
-
-def log_marginal_likelihood(self, prior_precision: torch.Tensor | None = None, sigma_noise: torch.Tensor | None = None) -
-
-

Compute the Laplace approximation to the log marginal likelihood subject -to specific Hessian approximations that subclasses implement. -Requires that the Laplace approximation has been fit before. -The resulting torch.Tensor is differentiable in prior_precision and -sigma_noise if these have gradients enabled. -By passing prior_precision or sigma_noise, the current value is -overwritten. This is useful for iterating on the log marginal likelihood.

-

Parameters

-
-
prior_precision : torch.Tensor, optional
-
prior precision if should be changed from current prior_precision value
-
sigma_noise : torch.Tensor, optional
-
observation noise standard deviation if should be changed
-
-

Returns

-
-
log_marglik : torch.Tensor
-
 
-
-
-
-def predictive_samples(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], pred_type: PredType | str = PredType.GLM, n_samples: int = 100, diagonal_output: bool = False, generator: torch.Generator | None = None) -
-
-

Sample from the posterior predictive on input data x. -Can be used, for example, for Thompson sampling.

-

Parameters

-
-
x : torch.Tensor or MutableMapping
-
input data (batch_size, input_shape)
-
pred_type : {'glm', 'nn'}, default='glm'
-
type of posterior predictive, linearized GLM predictive or neural -network sampling predictive. The GLM predictive is consistent with -the curvature approximations used here.
-
n_samples : int
-
number of samples
-
diagonal_output : bool
-
whether to use a diagonalized glm posterior predictive on the outputs. -Only applies when pred_type='glm'.
-
generator : torch.Generator, optional
-
random number generator to control the samples (if sampling used)
-
-

Returns

-
-
samples : torch.Tensor
-
samples (n_samples, batch_size, output_shape)
-
-
-
-def functional_variance(self, Js: torch.Tensor) ‑> torch.Tensor -
-
-

Compute functional variance for the 'glm' predictive: -f_var[i] = Js[i] @ P.inv() @ Js[i].T, which is a output x output -predictive covariance matrix. -Mathematically, we have for a single Jacobian -\mathcal{J} = \nabla_\theta f(x;\theta)\vert_{\theta_{MAP}} -the output covariance matrix - \mathcal{J} P^{-1} \mathcal{J}^T .

-

Parameters

-
-
Js : torch.Tensor
-
Jacobians of model output wrt parameters -(batch, outputs, parameters)
-
-

Returns

-
-
f_var : torch.Tensor
-
output covariance (batch, outputs, outputs)
-
-
-
-def functional_covariance(self, Js: torch.Tensor) ‑> torch.Tensor -
-
-

Compute functional covariance for the 'glm' predictive: -f_cov = Js @ P.inv() @ Js.T, which is a batchoutput x batchoutput -predictive covariance matrix.

-

This emulates the GP posterior covariance N([f(x1), …,f(xm)], Cov[f(x1), …, f(xm)]). -Useful for joint predictions, such as in batched Bayesian optimization.

-

Parameters

-
-
Js : torch.Tensor
-
Jacobians of model output wrt parameters -(batch*outputs, parameters)
-
-

Returns

-
-
f_cov : torch.Tensor
-
output covariance (batch*outputs, batch*outputs)
-
-
-
-def sample(self, n_samples: int = 100, generator: torch.Generator | None = None) -
-
-

Sample from the Laplace posterior approximation, i.e., - \theta \sim \mathcal{N}(\theta_{MAP}, P^{-1}).

-

Parameters

-
-
n_samples : int, default=100
-
number of samples
-
generator : torch.Generator, optional
-
random number generator to control the samples
-
-

Returns

-
-
samples : torch.Tensor
-
 
-
-
-
-def state_dict(self) ‑> dict[str, typing.Any] -
-
-
-
-
-def load_state_dict(self, state_dict: dict[str, Any]) ‑> None -
-
-
-
-
-

Inherited members

- -
-
-class FullLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, backend_kwargs: dict[str, Any] | None = None) -
-
-

Laplace approximation with full, i.e., dense, log likelihood Hessian approximation -and hence posterior precision. Based on the chosen backend parameter, the full -approximation can be, for example, a generalized Gauss-Newton matrix. -Mathematically, we have P \in \mathbb{R}^{P \times P}. -See BaseLaplace for the full interface.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop posterior_scale : torch.Tensor
-
-

Posterior scale (square root of the covariance), i.e., -P^{-\frac{1}{2}}.

-

Returns

-
-
scale : torch.tensor
-
(parameters, parameters)
-
-
-
prop posterior_covariance : torch.Tensor
-
-

Posterior covariance, i.e., P^{-1}.

-

Returns

-
-
covariance : torch.tensor
-
(parameters, parameters)
-
-
-
prop posterior_precision : torch.Tensor
-
-

Posterior precision P.

-

Returns

-
-
precision : torch.tensor
-
(parameters, parameters)
-
-
-
-

Inherited members

- -
-
-class KronLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, damping: bool = False, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Laplace approximation with Kronecker factored log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have for each parameter group, e.g., torch.nn.Module, -that \P\approx Q \otimes H. -See BaseLaplace for the full interface and see -Kron and KronDecomposed for the structure of -the Kronecker factors. Kron is used to aggregate factors by summing up and -KronDecomposed is used to add the prior, a Hessian factor (e.g. temperature), -and computing posterior covariances, marginal likelihood, etc. -Damping can be enabled by setting damping=True.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop posterior_precision : KronDecomposed
-
-

Kronecker factored Posterior precision P.

-

Returns

-
-
precision : KronDecomposed
-
 
-
-
-
prop prior_precision : torch.Tensor
-
-
-
-
-

Methods

-
-
-def state_dict(self) ‑> dict[str, typing.Any] -
-
-
-
-
-def load_state_dict(self, state_dict: dict[str, Any]) -
-
-
-
-
-

Inherited members

- -
-
-class DiagLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Laplace approximation with diagonal log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have P \approx \textrm{diag}(P). -See BaseLaplace for the full interface.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop posterior_precision : torch.Tensor
-
-

Diagonal posterior precision p.

-

Returns

-
-
precision : torch.tensor
-
(parameters)
-
-
-
prop posterior_scale : torch.Tensor
-
-

Diagonal posterior scale \sqrt{p^{-1}}.

-

Returns

-
-
precision : torch.tensor
-
(parameters)
-
-
-
prop posterior_variance : torch.Tensor
-
-

Diagonal posterior variance p^{-1}.

-

Returns

-
-
precision : torch.tensor
-
(parameters)
-
-
-
-

Inherited members

- -
-
-class FunctionalLaplace -(model: nn.Module, likelihood: Likelihood | str, n_subset: int, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, dict_key_x='input_ids', dict_key_y='labels', backend: type[CurvatureInterface] | None = laplace.curvature.backpack.BackPackGGN, backend_kwargs: dict[str, Any] | None = None, independent_outputs: bool = False, seed: int = 0) -
-
-

Applying the GGN (Generalized Gauss-Newton) approximation for the Hessian in the Laplace approximation of the posterior -turns the underlying probabilistic model from a BNN into a GLM (generalized linear model). -This GLM (in the weight space) is equivalent to a GP (in the function space), see -Approximate Inference Turns Deep Networks into Gaussian Processes (Khan et al., 2019)

-

This class implements the (approximate) GP inference through which -we obtain the desired quantities (posterior predictive, marginal log-likelihood). -See Improving predictions of Bayesian neural nets via local linearization (Immer et al., 2021) -for more details.

-

Note that for likelihood='classification', we approximate L_{NN} with a diagonal matrix -( L_{NN} is a block-diagonal matrix, where blocks represent Hessians of per-data-point log-likelihood w.r.t. -neural network output f , See Appendix A.2.1 for exact definition). We -resort to such an approximation because of the (possible) errors found in Laplace approximation for -multiclass GP classification in Chapter 3.5 of R&W 2006 GP book, -see the question -here -for more details. Alternatively, one could also resort to one-vs-one or one-vs-rest implementations -for multiclass classification, however, that is not (yet) supported here.

-

Parameters

-
-
num_data : int
-
number of data points for Subset-of-Data (SOD) approximate GP inference.
-
diagonal_kernel : bool
-
GP kernel here is product of Jacobians, which results in a C \times C matrix where C is the output -dimension. If diagonal_kernel=True, only a diagonal of a GP kernel is used. This is (somewhat) equivalent to -assuming independent GPs across output channels.
-
-

See BaseLaplace class for the full interface.

-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop gp_kernel_prior_variance
-
-
-
-
prop log_det_ratio : torch.Tensor
-
-

Computes log determinant term in GP marginal likelihood

-

For classification we use eq. (3.44) from Chapter 3.5 from -GP book R&W 2006 with -(note that we always use diagonal approximation D of the Hessian of log likelihood w.r.t. f):

-

log determinant term := \log | I + D^{1/2}K D^{1/2} |

-

For regression, we use "standard" GP marginal likelihood:

-

log determinant term := \log | K + \sigma_2 I |

-
-
prop scatter : torch.Tensor
-
-

Compute scatter term in GP log marginal likelihood.

-

For classification we use eq. (3.44) from Chapter 3.5 from -GP book R&W 2006 with \hat{f} = f :

-

scatter term := f K^{-1} f^{T}

-

For regression, we use "standard" GP marginal likelihood:

-

scatter term := (y - m)K^{-1}(y -m )^T , -where m is the mean of the GP prior, which in our case corresponds to - m := f + J (\theta - \theta_{MAP})

-
-
prop prior_precision
-
-
-
-
-

Methods

-
-
-def fit(self, train_loader: DataLoader | MutableMapping, progress_bar: bool = False) -
-
-

Fit the Laplace approximation of a GP posterior.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
train_loader.dataset needs to be set to access N, size of the data set -train_loader.batch_size needs to be set to access b batch_size
-
progress_bar : bool
-
whether to show a progress bar during the fitting process.
-
-
-
-def predictive_samples(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any], pred_type: PredType | str = PredType.GLM, n_samples: int = 100, diagonal_output: bool = False, generator: torch.Generator | None = None) -
-
-

Sample from the posterior predictive on input data x. -Can be used, for example, for Thompson sampling.

-

Parameters

-
-
x : torch.Tensor or MutableMapping
-
input data (batch_size, input_shape)
-
pred_type : {'glm'}, default='glm'
-
type of posterior predictive, linearized GLM predictive.
-
n_samples : int
-
number of samples
-
diagonal_output : bool
-
whether to use a diagonalized glm posterior predictive on the outputs. -Only applies when pred_type='glm'.
-
generator : torch.Generator, optional
-
random number generator to control the samples (if sampling used)
-
-

Returns

-
-
samples : torch.Tensor
-
samples (n_samples, batch_size, output_shape)
-
-
-
-def functional_variance(self, Js_star: torch.Tensor) ‑> torch.Tensor -
-
-

GP posterior variance:

-

k_{**} - K_{*M} (K_{MM}+ L_{MM}^{-1})^{-1} K_{M*}

-

Parameters

-
-
Js_star : torch.Tensor of shape (N*, C, P)
-
Jacobians of test data points
-
-

Returns

-
-
f_var : torch.Tensor of shape (N*,C, C)
-
Contains the posterior variances of N* testing points.
-
-
-
-def functional_covariance(self, Js_star: torch.Tensor) ‑> torch.Tensor -
-
-

GP posterior covariance:

-

k_{**} - K_{*M} (K_{MM}+ L_{MM}^{-1})^{-1} K_{M*}

-

Parameters

-
-
Js_star : torch.Tensor of shape (N*, C, P)
-
Jacobians of test data points
-
-

Returns

-
-
f_var : torch.Tensor of shape (N*xC, N*xC)
-
Contains the posterior covariances of N* testing points.
-
-
-
-def optimize_prior_precision(self, pred_type: PredType | str = PredType.GP, method: TuningMethod | str = TuningMethod.MARGLIK, n_steps: int = 100, lr: float = 0.1, init_prior_prec: float | torch.Tensor = 1.0, prior_structure: PriorStructure | str = PriorStructure.SCALAR, val_loader: DataLoader | None = None, loss: torchmetrics.Metric | Callable[[torch.Tensor], torch.Tensor | float] | None = None, log_prior_prec_min: float = -4, log_prior_prec_max: float = 4, grid_size: int = 100, link_approx: LinkApprox | str = LinkApprox.PROBIT, n_samples: int = 100, verbose: bool = False, progress_bar: bool = False) -
-
-

optimize_prior_precision_base from BaseLaplace with pred_type='gp'

-
-
-def log_marginal_likelihood(self, prior_precision: torch.Tensor | None = None, sigma_noise: torch.Tensor | None = None) -
-
-

Compute the Laplace approximation to the log marginal likelihood. -Requires that the Laplace approximation has been fit before. -The resulting torch.Tensor is differentiable in prior_precision and -sigma_noise if these have gradients enabled. -By passing prior_precision or sigma_noise, the current value is -overwritten. This is useful for iterating on the log marginal likelihood.

-

Parameters

-
-
prior_precision : torch.Tensor, optional
-
prior precision if should be changed from current prior_precision value
-
sigma_noise : torch.Tensor, optional
-
observation noise standard deviation if should be changed
-
-

Returns

-
-
log_marglik : torch.Tensor
-
 
-
-
-
-def state_dict(self) ‑> dict -
-
-
-
-
-def load_state_dict(self, state_dict: dict) -
-
-
-
-
-

Inherited members

- -
-
-class LowRankLaplace -(model: nn.Module, likelihood: Likelihood | str, backend: type[CurvatureInterface] = laplace.curvature.curvature.CurvatureInterface, sigma_noise: float | torch.Tensor = 1, prior_precision: float | torch.Tensor = 1, prior_mean: float | torch.Tensor = 0, temperature: float = 1, enable_backprop: bool = False, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend_kwargs: dict[str, Any] | None = None) -
-
-

Laplace approximation with low-rank log likelihood Hessian (approximation). -The low-rank matrix is represented by an eigendecomposition (vecs, values). -Based on the chosen backend, either a true Hessian or, for example, GGN -approximation could be used. -The posterior precision is computed as - P = V diag(l) V^T + P_0. -To sample, compute the functional variance, and log determinant, algebraic tricks -are usedto reduce the costs of inversion to the that of a K -imes K matrix -if we have a rank of K.

-

Note that only AsdfghjklHessian backend is supported. Install it via: -pip install git+https://git@github.com/wiseodd/asdl@asdfghjkl

-

See BaseLaplace for the full interface.

-

Ancestors

- -

Instance variables

-
-
prop V : torch.Tensor
-
-
-
-
prop Kinv : torch.Tensor
-
-
-
-
prop posterior_precision : tuple[tuple[torch.Tensor, torch.Tensor], torch.Tensor]
-
-

Return correctly scaled posterior precision that would be constructed -as H[0] @ diag(H[1]) @ H[0].T + self.prior_precision_diag.

-

Returns

-
-
H : tuple(eigenvectors, eigenvalues)
-
scaled self.H with temperature and loss factors.
-
prior_precision_diag : torch.Tensor
-
diagonal prior precision shape parameters to be added to H.
-
-
-
-

Inherited members

- -
-
-class LLLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, last_layer_name: str | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Baseclass for all last-layer Laplace approximations in this library. -Subclasses specify the structure of the Hessian approximation. -See BaseLaplace for the full interface.

-

A Laplace approximation is represented by a MAP which is given by the -model parameter and a posterior precision or covariance specifying -a Gaussian distribution \mathcal{N}(\theta_{MAP}, P^{-1}). -Here, only the parameters of the last layer of the neural network -are treated probabilistically. -The goal of this class is to compute the posterior precision P -which sums as - -P = \sum_{n=1}^N \nabla^2_\theta \log p(\mathcal{D}_n \mid \theta) -\vert_{\theta_{MAP}} + \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}}. - -Every subclass implements different approximations to the log likelihood Hessians, -for example, a diagonal one. The prior is assumed to be Gaussian and therefore we have -a simple form for \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}} = P_0 . -In particular, we assume a scalar or diagonal prior precision so that in -all cases P_0 = \textrm{diag}(p_0) and the structure of p_0 can be varied.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
 
-
likelihood : Likelihood or {'classification', 'regression'}
-
determines the log likelihood Hessian approximation
-
sigma_noise : torch.Tensor or float, default=1
-
observation noise for the regression setting; must be 1 for classification
-
prior_precision : torch.Tensor or float, default=1
-
prior precision of a Gaussian prior (= weight decay); -can be scalar, per-layer, or diagonal in the most general case
-
prior_mean : torch.Tensor or float, default=0
-
prior mean of a Gaussian prior, useful for continual learning
-
temperature : float, default=1
-
temperature of the likelihood; lower temperature leads to more -concentrated posterior and vice versa.
-
enable_backprop : bool, default=False
-
whether to enable backprop to the input x through the Laplace predictive. -Useful for e.g. Bayesian optimization.
-
feature_reduction : FeatureReduction or str, optional, default=None
-
when the last-layer features is a tensor of dim >= 3, this tells how to reduce -it into a dim-2 tensor. E.g. in LLMs for non-language modeling problems, -the penultultimate output is a tensor of shape (batch_size, seq_len, embd_dim). -But the last layer maps (batch_size, embd_dim) to (batch_size, n_classes). -Note: Make sure that this option faithfully reflects the reduction in the model -definition. When inputting a string, available options are -{'pick_first', 'pick_last', 'average'}.
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
backend : subclasses of CurvatureInterface
-
backend for access to curvature/Hessian approximations
-
last_layer_name : str, default=None
-
name of the model's last layer, if None it will be determined automatically
-
backend_kwargs : dict, default=None
-
arguments passed to the backend on initialization, for example to -set the number of MC samples for stochastic approximations.
-
-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop prior_precision_diag : torch.Tensor
-
-

Obtain the diagonal prior precision p_0 constructed from either -a scalar or diagonal prior precision.

-

Returns

-
-
prior_precision_diag : torch.Tensor
-
 
-
-
-
-

Methods

-
-
-def fit(self, train_loader: DataLoader, override: bool = True, progress_bar: bool = False) ‑> None -
-
-

Fit the local Laplace approximation at the parameters of the model.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
each iterate is a training batch, either (X, y) tensors or a dict-like -object containing keys as expressed by self.dict_key_x and -self.dict_key_y. train_loader.dataset needs to be set to access -N, size of the data set.
-
override : bool, default=True
-
whether to initialize H, loss, and n_data again; setting to False is useful for -online learning settings to accumulate a sequential posterior approximation.
-
progress_bar : bool, default=False
-
 
-
-
-
-def functional_variance_fast(self, X) -
-
-

Should be overriden if there exists a trick to make this fast!

-

Parameters

-
-
X : torch.Tensor of shape (batch_size, input_dim)
-
 
-
-

Returns

-
-
f_var_diag : torch.Tensor of shape (batch_size, num_outputs)
-
Corresponding to the diagonal of the covariance matrix of the outputs
-
-
-
-def state_dict(self) ‑> dict[str, typing.Any] -
-
-
-
-
-def load_state_dict(self, state_dict: dict[str, Any]) ‑> None -
-
-
-
-
-

Inherited members

- -
-
-class FullLLLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, last_layer_name: str | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Last-layer Laplace approximation with full, i.e., dense, log likelihood Hessian approximation -and hence posterior precision. Based on the chosen backend parameter, the full -approximation can be, for example, a generalized Gauss-Newton matrix. -Mathematically, we have P \in \mathbb{R}^{P \times P}. -See FullLaplace, LLLaplace, and BaseLaplace for the full interface.

-

Ancestors

- -

Inherited members

- -
-
-class KronLLLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, last_layer_name: str | None = None, damping: bool = False, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Last-layer Laplace approximation with Kronecker factored log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have for the last parameter group, i.e., torch.nn.Linear, -that \P\approx Q \otimes H. -See KronLaplace, LLLaplace, and BaseLaplace for the full interface and see -Kron and KronDecomposed for the structure of -the Kronecker factors. Kron is used to aggregate factors by summing up and -KronDecomposed is used to add the prior, a Hessian factor (e.g. temperature), -and computing posterior covariances, marginal likelihood, etc. -Use of damping is possible by initializing or setting damping=True.

-

Ancestors

- -

Inherited members

- -
-
-class DiagLLLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, last_layer_name: str | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Last-layer Laplace approximation with diagonal log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have P \approx \textrm{diag}(P). -See DiagLaplace, LLLaplace, and BaseLaplace for the full interface.

-

Ancestors

- -

Inherited members

- -
-
-class FunctionalLLLaplace -(model: nn.Module, likelihood: Likelihood | str, n_subset: int, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', last_layer_name: str | None = None, backend: type[CurvatureInterface] | None = laplace.curvature.backpack.BackPackGGN, backend_kwargs: dict[str, Any] | None = None, independent_outputs: bool = False, seed: int = 0) -
-
-

Here not much changes in terms of GP inference compared to FunctionalLaplace class. -Since now we treat only the last layer probabilistically and the rest of the network is used as a "fixed feature -extractor", that means that the X \in \mathbb{R}^{M \times D} in GP inference changes -to \tilde{X} \in \mathbb{R}^{M \times l_{n-1}} , -where l_{n-1} is the dimension of the output -of the penultimate NN layer.

-

See FunctionalLaplace for the full interface.

-

Ancestors

- -

Methods

-
-
-def fit(self, train_loader: DataLoader) ‑> None -
-
-

Fit the Laplace approximation of a GP posterior.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
train_loader.dataset needs to be set to access N, size of the data set -train_loader.batch_size needs to be set to access b batch_size
-
-
-
-def state_dict(self) ‑> dict -
-
-
-
-
-def load_state_dict(self, state_dict: dict) -
-
-
-
-
-

Inherited members

- -
-
-class SubnetLaplace -(model: nn.Module, likelihood: Likelihood | str, subnetwork_indices: torch.LongTensor, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, backend: Type[CurvatureInterface] | None = None, backend_kwargs: dict | None = None, asdl_fisher_kwargs: dict | None = None) -
-
-

Class for subnetwork Laplace, which computes the Laplace approximation over just a subset -of the model parameters (i.e. a subnetwork within the neural network), as proposed in [1]. -Subnetwork Laplace can only be used with either a full or a diagonal Hessian approximation.

-

A Laplace approximation is represented by a MAP which is given by the -model parameter and a posterior precision or covariance specifying -a Gaussian distribution \mathcal{N}(\theta_{MAP}, P^{-1}). -Here, only a subset of the model parameters (i.e. a subnetwork of the -neural network) are treated probabilistically. -The goal of this class is to compute the posterior precision P -which sums as - -P = \sum_{n=1}^N \nabla^2_\theta \log p(\mathcal{D}_n \mid \theta) -\vert_{\theta_{MAP}} + \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}}. - -The prior is assumed to be Gaussian and therefore we have a simple form for -\nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}} = P_0 . -In particular, we assume a scalar or diagonal prior precision so that in -all cases P_0 = \textrm{diag}(p_0) and the structure of p_0 can be varied.

-

The subnetwork Laplace approximation only supports a full, i.e., dense, log likelihood -Hessian approximation and hence posterior precision. -Based on the chosen backend -parameter, the full approximation can be, for example, a generalized Gauss-Newton -matrix. -Mathematically, we have P \in \mathbb{R}^{P \times P}. -See FullLaplace and BaseLaplace for the full interface.

-

References

-

[1] Daxberger, E., Nalisnick, E., Allingham, JU., Antorán, J., Hernández-Lobato, JM. -Bayesian Deep Learning via Subnetwork Inference. -ICML 2021.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
 
-
likelihood : {'classification', 'regression'}
-
determines the log likelihood Hessian approximation
-
subnetwork_indices : torch.LongTensor
-
indices of the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters())) -that define the subnetwork to apply the Laplace approximation over
-
sigma_noise : torch.Tensor or float, default=1
-
observation noise for the regression setting; must be 1 for classification
-
prior_precision : torch.Tensor or float, default=1
-
prior precision of a Gaussian prior (= weight decay); -can be scalar, per-layer, or diagonal in the most general case
-
prior_mean : torch.Tensor or float, default=0
-
prior mean of a Gaussian prior, useful for continual learning
-
temperature : float, default=1
-
temperature of the likelihood; lower temperature leads to more -concentrated posterior and vice versa.
-
backend : subclasses of CurvatureInterface
-
backend for access to curvature/Hessian approximations
-
backend_kwargs : dict, default=None
-
arguments passed to the backend on initialization, for example to -set the number of MC samples for stochastic approximations.
-
-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop prior_precision_diag : torch.Tensor
-
-

Obtain the diagonal prior precision p_0 constructed from either -a scalar or diagonal prior precision.

-

Returns

-
-
prior_precision_diag : torch.Tensor
-
 
-
-
-
prop mean_subnet : torch.Tensor
-
-
-
-
-

Methods

-
-
-def assemble_full_samples(self, subnet_samples) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class FullSubnetLaplace -(model: nn.Module, likelihood: Likelihood | str, subnetwork_indices: torch.LongTensor, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, backend: Type[CurvatureInterface] | None = None, backend_kwargs: dict | None = None, asdl_fisher_kwargs: dict | None = None) -
-
-

Subnetwork Laplace approximation with full, i.e., dense, log likelihood Hessian -approximation and hence posterior precision. Based on the chosen backend parameter, -the full approximation can be, for example, a generalized Gauss-Newton matrix. -Mathematically, we have P \in \mathbb{R}^{P \times P}. -See FullLaplace, SubnetLaplace, and BaseLaplace for the full interface.

-

Ancestors

- -

Inherited members

- -
-
-class DiagSubnetLaplace -(model: nn.Module, likelihood: Likelihood | str, subnetwork_indices: torch.LongTensor, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, backend: Type[CurvatureInterface] | None = None, backend_kwargs: dict | None = None, asdl_fisher_kwargs: dict | None = None) -
-
-

Subnetwork Laplace approximation with diagonal log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have P \approx \textrm{diag}(P). -See DiagLaplace, SubnetLaplace, and BaseLaplace for the full interface.

-

Ancestors

- -

Inherited members

- -
-
-class SubsetOfWeights -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var ALL
-
-
-
-
var LAST_LAYER
-
-
-
-
var SUBNETWORK
-
-
-
-
-
-
-class HessianStructure -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var FULL
-
-
-
-
var KRON
-
-
-
-
var DIAG
-
-
-
-
var LOWRANK
-
-
-
-
var GP
-
-
-
-
-
-
-class Likelihood -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var REGRESSION
-
-
-
-
var CLASSIFICATION
-
-
-
-
var REWARD_MODELING
-
-
-
-
-
-
-class PredType -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var GLM
-
-
-
-
var NN
-
-
-
-
var GP
-
-
-
-
-
-
-class LinkApprox -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var MC
-
-
-
-
var PROBIT
-
-
-
-
var BRIDGE
-
-
-
-
var BRIDGE_NORM
-
-
-
-
-
-
-class TuningMethod -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var MARGLIK
-
-
-
-
var GRIDSEARCH
-
-
-
-
-
-
-class PriorStructure -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var SCALAR
-
-
-
-
var DIAG
-
-
-
-
var LAYERWISE
-
-
-
-
-
-
-
-
- -
- - - diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..c8ca64f3 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,498 @@ +
+ Laplace + +
+ +The laplace package facilitates the application of Laplace approximations for entire neural networks, subnetworks of neural networks, or just their last layer. +The package enables posterior approximations, marginal-likelihood estimation, and various posterior predictive computations. +The library documentation is available at [https://aleximmer.github.io/Laplace](https://aleximmer.github.io/Laplace). + +There is also a corresponding paper, [_Laplace Redux — Effortless Bayesian Deep Learning_](https://arxiv.org/abs/2106.14806), which introduces the library, provides an introduction to the Laplace approximation, reviews its use in deep learning, and empirically demonstrates its versatility and competitiveness. Please consider referring to the paper when using our library: + +```bibtex +@inproceedings{laplace2021, + title={Laplace Redux--Effortless {B}ayesian Deep Learning}, + author={Erik Daxberger and Agustinus Kristiadi and Alexander Immer + and Runa Eschenhagen and Matthias Bauer and Philipp Hennig}, + booktitle={{N}eur{IPS}}, + year={2021} +} +``` + +The [code](https://github.com/runame/laplace-redux) to reproduce the experiments in the paper is also publicly available; it provides examples of how to use our library for predictive uncertainty quantification, model selection, and continual learning. + +!!! important + + As a user, one should not expect Laplace to work automatically. + That is, one should experiment with different Laplace's options + (Hessian factorization, prior precision tuning method, predictive method, backend, + etc!). Try looking at various papers that use Laplace for references on how to + set all those options depending on the applications/problems at hand. + +## Setup + +!!! important + + We assume Python >= 3.9 since lower versions are [(soon to be) deprecated](https://devguide.python.org/versions/). + PyTorch version 2.0 and up is also required for full compatibility. + +To install laplace with `pip`, run the following: + +```bash +pip install laplace-torch +``` + +Additionally, if you want to use the `asdfghjkl` backend, please install it via: + +```bash +pip install git+https://git@github.com/wiseodd/asdl@asdfghjkl +``` + +### Setup dev environment + +For development purposes, e.g. if you would like to make contributions, follow +the following steps: + +**With `uv`** + +1. Install [`uv`](https://github.com/astral-sh/uv), e.g. `pip install --upgrade uv` +2. Then clone this repository and install the development dependencies: + +```bash +git clone git@github.com:aleximmer/Laplace.git +uv sync --all-extras +``` + +3. `laplace-torch` is now available in editable mode, e.g. you can run: + +```bash +uv run python examples/regression_example.py + +# Or, equivalently: +source .venv/bin/activate +python examples/regression_example.py +``` + +**With `pip`** + +```bash +git clone git@github.com:aleximmer/Laplace.git + +# Recommended to create a virtualenv before the following step +pip install -e ".[dev]" + +# Run as usual, e.g. +python examples/regression_examples.py +``` + +!!! note + + See [contributing guideline](#contributing). + We're looking forward to your contributions! + +## Quickstart + +### Simple usage + +In the following example, a pre-trained model is loaded, +then the Laplace approximation is fit to the training data +(using a diagonal Hessian approximation over all parameters), +and the prior precision is optimized with cross-validation `"gridsearch"`. +After that, the resulting LA is used for prediction with +the `"probit"` predictive for classification. + +!!! important + + Laplace expects all data loaders, e.g. `train_loader` and `val_loader` below, + to be instances of PyTorch + [`DataLoader`](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html). + Each batch, `next(iter(data_loader))` must either be the standard `(X, y)` tensors + or a dict-like object containing at least the keys specified in + `dict_key_x` and `dict_key_y` in Laplace's constructor. + +!!! important + + The total number of data points in all data loaders must be accessible via + `len(train_loader.dataset)`. + +!!! important + + In `optimize_prior_precision`, make sure to match the arguments with + the ones you want to pass in `la(x, ...)` during prediction. + +```python +from laplace import Laplace + +# Pre-trained model +model = load_map_model() + +# User-specified LA flavor +la = Laplace(model, "classification", + subset_of_weights="all", + hessian_structure="diag") +la.fit(train_loader) +la.optimize_prior_precision( + method="gridsearch", + pred_type="glm", + link_approx="probit", + val_loader=val_loader +) + +# User-specified predictive approx. +pred = la(x, pred_type="glm", link_approx="probit") +``` + +### Marginal likelihood + +The marginal likelihood can be used for model selection [10] and is differentiable +for continuous hyperparameters like the prior precision or observation noise. +Here, we fit the library default, KFAC last-layer LA and differentiate +the log marginal likelihood. + +```python +from laplace import Laplace + +# Un- or pre-trained model +model = load_model() + +# Default to recommended last-layer KFAC LA: +la = Laplace(model, likelihood="regression") +la.fit(train_loader) + +# ML w.r.t. prior precision and observation noise +ml = la.log_marginal_likelihood(prior_prec, obs_noise) +ml.backward() +``` + +### Laplace on LLMs + +!!! tip + + This library also supports Huggingface models and parameter-efficient fine-tuning. + See [Huggingface LLM example](huggingface_example.md) for the full exposition. + +First, we need to wrap the pretrained model so that the `forward` method takes a +dict-like input. Note that when you iterate over a Huggingface dataloader, +this is what you get by default. Having a dict-like input is nice since different models +have different number of inputs (e.g. GPT-like LLMs only take `input_ids`, while BERT-like +ones take both `input_ids` and `attention_mask`, etc.). Inside this `forward` method you +can do your usual preprocessing like moving the tensor inputs into the correct device. + +```python +class MyGPT2(nn.Module): + def __init__(self, tokenizer: PreTrainedTokenizer) - None: + super().__init__() + config = GPT2Config.from_pretrained("gpt2") + config.pad_token_id = tokenizer.pad_token_id + config.num_labels = 2 + self.hf_model = GPT2ForSequenceClassification.from_pretrained( + "gpt2", config=config + ) + + def forward(self, data: MutableMapping) - torch.Tensor: + device = next(self.parameters()).device + input_ids = data["input_ids"].to(device) + attn_mask = data["attention_mask"].to(device) + output_dict = self.hf_model(input_ids=input_ids, attention_mask=attn_mask) + return output_dict.logits +``` + +Then you can "select" which parameters of the LLM you want to apply the Laplace approximation +on, by switching off the gradients of the "unneeded" parameters. +For example, we can replicate a last-layer Laplace: (in actual practice, use `Laplace(..., subset_of_weights='last_layer', ...)` instead, though!) + +```python +model = MyGPT2(tokenizer) +model.eval() + +# Enable grad only for the last layer +for p in model.hf_model.parameters(): + p.requires_grad = False +for p in model.hf_model.score.parameters(): + p.requires_grad = True + +la = Laplace( + model, + likelihood="classification", + # Will only hit the last-layer since it's the only one that is grad-enabled + subset_of_weights="all", + hessian_structure="diag", +) +la.fit(dataloader) +la.optimize_prior_precision() + +test_data = next(iter(dataloader)) +pred = la(test_data) +``` + +This is useful because we can apply the LA only on the parameter-efficient finetuning +weights. E.g., we can fix the LLM itself, and apply the Laplace approximation only +on the LoRA weights. Huggingface will automatically switch off the non-LoRA weights' +gradients. + +```python +def get_lora_model(): + model = MyGPT2(tokenizer) # Note we don't disable grad + config = LoraConfig( + r=4, + lora_alpha=16, + target_modules=["c_attn"], # LoRA on the attention weights + lora_dropout=0.1, + bias="none", + ) + lora_model = get_peft_model(model, config) + return lora_model + +lora_model = get_lora_model() + +# Train it as usual here... + +lora_model.eval() + +lora_la = Laplace( + lora_model, + likelihood="classification", + subset_of_weights="all", + hessian_structure="diag", + backend=AsdlGGN, +) + +test_data = next(iter(dataloader)) +lora_pred = lora_la(test_data) +``` + +### Subnetwork Laplace + +This example shows how to fit the Laplace approximation over only +a subnetwork within a neural network (while keeping all other parameters +fixed at their MAP estimates), as proposed in [11]. It also exemplifies +different ways to specify the subnetwork to perform inference over. + +First, we make use of `SubnetLaplace`, where we specify the subnetwork by +generating a list of indices for the active model parameters. + +```python +from laplace import Laplace + +# Pre-trained model +model = load_model() + +# Examples of different ways to specify the subnetwork +# via indices of the vectorized model parameters +# +# Example 1: select the 128 parameters with the largest magnitude +from laplace.utils import LargestMagnitudeSubnetMask +subnetwork_mask = LargestMagnitudeSubnetMask(model, n_params_subnet=128) +subnetwork_indices = subnetwork_mask.select() + +# Example 2: specify the layers that define the subnetwork +from laplace.utils import ModuleNameSubnetMask +subnetwork_mask = ModuleNameSubnetMask(model, module_names=["layer.1", "layer.3"]) +subnetwork_mask.select() +subnetwork_indices = subnetwork_mask.indices + +# Example 3: manually define the subnetwork via custom subnetwork indices +import torch +subnetwork_indices = torch.tensor([0, 4, 11, 42, 123, 2021]) + +# Define and fit subnetwork LA using the specified subnetwork indices +la = Laplace(model, "classification", + subset_of_weights="subnetwork", + hessian_structure="full", + subnetwork_indices=subnetwork_indices) +la.fit(train_loader) +``` + +Besides `SubnetLaplace`, you can, as already mentioned, also treat the last +layer only using `Laplace(..., subset_of_weights='last_layer')`, which uses +`LLLaplace`. As a third method, you may define a subnetwork by disabling +gradients of fixed model parameters. The different methods target different use +cases. Each method has pros and cons, please see [this +discussion](https://github.com/aleximmer/Laplace/issues/217#issuecomment-2278311460) +for details. In summary + +- Disable-grad: General method to perform Laplace on specific types of + layer/parameter, e.g. in an LLM with LoRA. Can be used to emulate `LLLaplace` + as well. Always use `subset_of_weights='all'` for this method. + - subnet selection by disabling grads is more efficient than + `SubnetLaplace` since it avoids calculating full Jacobians first + - disabling grads can only be performed on `Parameter` level and not for + individual weights, so this doesn't cover all cases that `SubnetLaplace` + offers such as `Largest*SubnetMask` or `RandomSubnetMask` +- `LLLaplace`: last-layer specific code with improved performance (#145) +- `SubnetLaplace`: more fine-grained partitioning such as + `LargestMagnitudeSubnetMask` + +### Serialization + +As with plain `torch`, we support to ways to serialize data. + +One is the familiar `state_dict` approach. Here you need to save and re-create +both `model` and `Laplace`. Use this for long-term storage of models and +sharing of a fitted `Laplace` instance. + +```py +# Save model and Laplace instance +torch.save(model.state_dict(), "model_state_dict.bin") +torch.save(la.state_dict(), "la_state_dict.bin") + +# Load serialized data +model2 = MyModel(...) +model2.load_state_dict(torch.load("model_state_dict.bin")) +la2 = Laplace(model2, "classification", + subset_of_weights="all", + hessian_structure="diag") +la2.load_state_dict(torch.load("la_state_dict.bin")) +``` + +The second approach is to save the whole `Laplace` object, including +`self.model`. This is less verbose and more convenient since you have the +trained model and the fitted `Laplace` data stored in one place, but [also comes with +some +drawbacks](https://pytorch.org/tutorials/beginner/saving_loading_models.html#saving-loading-model-for-inference). +Use this for quick save-load cycles during experiments, say. + +```py +# Save Laplace, including la.model +torch.save(la, "la.pt") + +# Load both +torch.load("la.pt") +``` + +Some Laplace variants such as `LLLaplace` might have trouble being serialized +using the default `pickle` module, which `torch.save()` and `torch.load()` use +(`AttributeError: Can't pickle local object ...`). In this case, the +[`dill`](https://github.com/uqfoundation/dill) package will come in handy. + +```py +import dill + +torch.save(la, "la.pt", pickle_module=dill) +``` + +With both methods, you are free to switch devices, for instance when you +trained on a GPU but want to run predictions on CPU. In this case, use + +```py +torch.load(..., map_location="cpu") +``` + +!!! warning + + Currently, this library always assumes that the model has an + output tensor of shape `(batch_size, ..., n_classes)`, so in + the case of image outputs, you need to rearrange from NCHW to NHWC. + +## Structure + +The laplace package consists of two main components: + +1. The subclasses of [`laplace.BaseLaplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/baselaplace.py) that implement different sparsity structures: different subsets of weights (`'all'`, `'subnetwork'` and `'last_layer'`) and different structures of the Hessian approximation (`'full'`, `'kron'`, `'lowrank'`, `'diag'` and `'gp'`). This results in _ten_ currently available options: `laplace.FullLaplace`, `laplace.KronLaplace`, `laplace.DiagLaplace`, `laplace.FunctionalLaplace` the corresponding last-layer variations `laplace.FullLLLaplace`, `laplace.KronLLLaplace`, `laplace.DiagLLLaplace` and `laplace.FunctionalLLLaplace` (which are all subclasses of [`laplace.LLLaplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/lllaplace.py)), [`laplace.SubnetLaplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/subnetlaplace.py) (which only supports `'full'` and `'diag'` Hessian approximations) and `laplace.LowRankLaplace` (which only supports inference over `'all'` weights). All of these can be conveniently accessed via the [`laplace.Laplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/laplace.py) function. +2. The backends in [`laplace.curvature`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/) which provide access to Hessian approximations of + the corresponding sparsity structures, for example, the diagonal GGN. + +Additionally, the package provides utilities for +decomposing a neural network into feature extractor and last layer for `LLLaplace` subclasses ([`laplace.utils.feature_extractor`](https://github.com/AlexImmer/Laplace/blob/main/laplace/utils/feature_extractor.py)) +and +effectively dealing with Kronecker factors ([`laplace.utils.matrix`](https://github.com/AlexImmer/Laplace/blob/main/laplace/utils/matrix.py)). + +Finally, the package implements several options to select/specify a subnetwork for `SubnetLaplace` (as subclasses of [`laplace.utils.subnetmask.SubnetMask`](https://github.com/AlexImmer/Laplace/blob/main/laplace/utils/subnetmask.py)). +Automatic subnetwork selection strategies include: uniformly at random (`laplace.utils.subnetmask.RandomSubnetMask`), by largest parameter magnitudes (`LargestMagnitudeSubnetMask`), and by largest marginal parameter variances (`LargestVarianceDiagLaplaceSubnetMask` and `LargestVarianceSWAGSubnetMask`). +In addition to that, subnetworks can also be specified manually, by listing the names of either the model parameters (`ParamNameSubnetMask`) or modules (`ModuleNameSubnetMask`) to perform Laplace inference over. + +## Extendability + +To extend the laplace package, new `BaseLaplace` subclasses can be designed, for example, +Laplace with a block-diagonal Hessian structure. +One can also implement custom subnetwork selection strategies as new subclasses of `SubnetMask`. + +Alternatively, extending or integrating backends (subclasses of [`curvature.curvature`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/curvature.py)) allows to provide different Hessian +approximations to the Laplace approximations. +For example, currently the [`curvature.CurvlinopsInterface`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/curvlinops.py) based on [Curvlinops](https://github.com/f-dangel/curvlinops) and the native `torch.func` (previously known as `functorch`), [`curvature.BackPackInterface`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/backpack.py) based on [BackPACK](https://github.com/f-dangel/backpack/) and [`curvature.AsdlInterface`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/asdl.py) based on [ASDL](https://github.com/kazukiosawa/asdfghjkl) are available. + +## When to use which backend + +!!! tip + + Each backend as its own caveat/behavior. The use the following to guide you + picking the suitable backend, depending on you model & application. + +- **Small, simple MLP, or last-layer Laplace:** Any backend should work well. + `CurvlinopsGGN` or `CurvlinopsEF` is recommended if + `hessian_factorization = 'kron'`, but it's inefficient for other factorizations. +- **LLMs with PEFT (e.g. LoRA):** `AsdlGGN` and `AsdlEF` are recommended. +- **Continuous Bayesian optimization:** `CurvlinopsGGN/EF` and `BackpackGGN/EF` are + recommended since they are the only ones supporting backprop over Jacobians. + +!!! caution + + The `curvlinops` backends are inefficient for full and diagonal factorizations. + Moreover, they're also inefficient for computing the Jacobians of large models + since they rely on `torch.func.jacrev` along `torch.func.vmap`! + Finally, `curvlinops` only computes K-FAC (`hessian_factorization = 'kron'`) + for `nn.Linear` and `nn.Conv2d` modules (including those inside larger modules + like Attention). + +!!! caution + + The `BackPack` backends are limited to models expressed as `nn.Sequential`. + Also, they're not compatible with normalization layers. + +## Documentation + +The documentation is available [here](https://aleximmer.github.io/Laplace) or can be generated and/or viewed locally: + +**With `uv`** + +```bash +# assuming the repository was cloned +uv sync --all-extras +# create docs and write to html +uv run bash update_docs.sh +# .. or serve the docs directly +uv run pdoc --http 0.0.0.0:8080 laplace --template-dir template +``` + +**With `pip`** + +```bash +# assuming the repository was cloned +pip install -e ".[dev]" +# create docs and write to html +bash update_docs.sh +# .. or serve the docs directly +pdoc --http 0.0.0.0:8080 laplace --template-dir template +``` + +## Contributing + +Pull requests are very welcome. Please follow these guidelines: + +1. Follow the [development setup](#setup-dev-environment). +2. Use [ruff](https://github.com/astral-sh/ruff) as autoformatter. Please refer to the following [makefile](https://github.com/aleximmer/Laplace/blob/main/makefile) and run it via `make ruff`. Please note that the order of `ruff check --fix` and `ruff format` is important! +3. Also use [ruff](https://github.com/astral-sh/ruff) as linter. Please manually fix all linting errors/warnings before opening a pull request. +4. Fully document your changes in the form of Python docstrings, typehinting, and (if applicable) code/markdown examples in the `./examples` subdirectory. +5. Provide as many test cases as possible. Make sure all test cases pass. + +Issues, bug reports, and ideas are also very welcome! + +## Useful links + +- Publishing package with `uv`: + +## References + +This package relies on various improvements to the Laplace approximation for neural networks, which was originally due to MacKay [1]. Please consider citing the respective papers if you use any of their proposed methods via our laplace library. + +- [1] MacKay, DJC. [_A Practical Bayesian Framework for Backpropagation Networks_](https://authors.library.caltech.edu/13793/). Neural Computation 1992. +- [2] Gibbs, M. N. [_Bayesian Gaussian Processes for Regression and Classification_](https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.147.1130&rep=rep1&type=pdf). PhD Thesis 1997. +- [3] Snoek, J., Rippel, O., Swersky, K., Kiros, R., Satish, N., Sundaram, N., Patwary, M., Prabhat, M., Adams, R. [_Scalable Bayesian Optimization Using Deep Neural Networks_](https://arxiv.org/abs/1502.05700). ICML 2015. +- [4] Ritter, H., Botev, A., Barber, D. [_A Scalable Laplace Approximation for Neural Networks_](https://openreview.net/forum?id=Skdvd2xAZ). ICLR 2018. +- [5] Foong, A. Y., Li, Y., Hernández-Lobato, J. M., Turner, R. E. [_'In-Between' Uncertainty in Bayesian Neural Networks_](https://arxiv.org/abs/1906.11537). ICML UDL Workshop 2019. +- [6] Khan, M. E., Immer, A., Abedi, E., Korzepa, M. [_Approximate Inference Turns Deep Networks into Gaussian Processes_](https://arxiv.org/abs/1906.01930). NeurIPS 2019. +- [7] Kristiadi, A., Hein, M., Hennig, P. [_Being Bayesian, Even Just a Bit, Fixes Overconfidence in ReLU Networks_](https://arxiv.org/abs/2002.10118). ICML 2020. +- [8] Immer, A., Korzepa, M., Bauer, M. [_Improving predictions of Bayesian neural nets via local linearization_](https://arxiv.org/abs/2008.08400). AISTATS 2021. +- [9] Sharma, A., Azizan, N., Pavone, M. [_Sketching Curvature for Efficient Out-of-Distribution Detection for Deep Neural Networks_](https://arxiv.org/abs/2102.12567). UAI 2021. +- [10] Immer, A., Bauer, M., Fortuin, V., Rätsch, G., Khan, EM. [_Scalable Marginal Likelihood Estimation for Model Selection in Deep Learning_](https://arxiv.org/abs/2104.04975). ICML 2021. +- [11] Daxberger, E., Nalisnick, E., Allingham, JU., Antorán, J., Hernández-Lobato, JM. [_Bayesian Deep Learning via Subnetwork Inference_](https://arxiv.org/abs/2010.14689). ICML 2021. diff --git a/docs/javascripts/mathjax.js b/docs/javascripts/mathjax.js new file mode 100644 index 00000000..5209b3c1 --- /dev/null +++ b/docs/javascripts/mathjax.js @@ -0,0 +1,19 @@ +window.MathJax = { + tex: { + inlineMath: [["\\(", "\\)"]], + displayMath: [["\\[", "\\]"]], + processEscapes: true, + processEnvironments: true, + }, + options: { + ignoreHtmlClass: ".*|", + processHtmlClass: "arithmatex", + }, +}; + +document$.subscribe(() => { + MathJax.startup.output.clearCache(); + MathJax.typesetClear(); + MathJax.texReset(); + MathJax.typesetPromise(); +}); diff --git a/docs/laplace.html b/docs/laplace.html deleted file mode 100644 index 86dd634e..00000000 --- a/docs/laplace.html +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - -laplace.laplace API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.laplace

-
-
-
-
-
-
-
-
-

Functions

-
-
-def Laplace(model: torch.nn.Module, likelihood: Likelihood | str, subset_of_weights: SubsetOfWeights | str = SubsetOfWeights.LAST_LAYER, hessian_structure: HessianStructure | str = HessianStructure.KRON, *args, **kwargs) -
-
-

Simplified Laplace access using strings instead of different classes.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
likelihood : Likelihood or str in {'classification', 'regression'}
-
 
-
subset_of_weights : SubsetofWeights or {'last_layer', 'subnetwork', 'all'}, default=SubsetOfWeights.LAST_LAYER
-
subset of weights to consider for inference
-
hessian_structure : HessianStructure or str in {'diag', 'kron', 'full', 'lowrank', 'gp'}, default=HessianStructure.KRON
-
structure of the Hessian approximation (note that in case of 'gp', -we are not actually doing any Hessian approximation, the inference is instead done in the functional space)
-
-

Returns

-
-
laplace : BaseLaplace
-
chosen subclass of BaseLaplace instantiated with additional arguments
-
-
-
-
-
-
-
- -
- - - diff --git a/docs/lllaplace.html b/docs/lllaplace.html deleted file mode 100644 index 1cb8f330..00000000 --- a/docs/lllaplace.html +++ /dev/null @@ -1,459 +0,0 @@ - - - - - - -laplace.lllaplace API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.lllaplace

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class LLLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, last_layer_name: str | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Baseclass for all last-layer Laplace approximations in this library. -Subclasses specify the structure of the Hessian approximation. -See BaseLaplace for the full interface.

-

A Laplace approximation is represented by a MAP which is given by the -model parameter and a posterior precision or covariance specifying -a Gaussian distribution \mathcal{N}(\theta_{MAP}, P^{-1}). -Here, only the parameters of the last layer of the neural network -are treated probabilistically. -The goal of this class is to compute the posterior precision P -which sums as - -P = \sum_{n=1}^N \nabla^2_\theta \log p(\mathcal{D}_n \mid \theta) -\vert_{\theta_{MAP}} + \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}}. - -Every subclass implements different approximations to the log likelihood Hessians, -for example, a diagonal one. The prior is assumed to be Gaussian and therefore we have -a simple form for \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}} = P_0 . -In particular, we assume a scalar or diagonal prior precision so that in -all cases P_0 = \textrm{diag}(p_0) and the structure of p_0 can be varied.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
 
-
likelihood : Likelihood or {'classification', 'regression'}
-
determines the log likelihood Hessian approximation
-
sigma_noise : torch.Tensor or float, default=1
-
observation noise for the regression setting; must be 1 for classification
-
prior_precision : torch.Tensor or float, default=1
-
prior precision of a Gaussian prior (= weight decay); -can be scalar, per-layer, or diagonal in the most general case
-
prior_mean : torch.Tensor or float, default=0
-
prior mean of a Gaussian prior, useful for continual learning
-
temperature : float, default=1
-
temperature of the likelihood; lower temperature leads to more -concentrated posterior and vice versa.
-
enable_backprop : bool, default=False
-
whether to enable backprop to the input x through the Laplace predictive. -Useful for e.g. Bayesian optimization.
-
feature_reduction : FeatureReduction or str, optional, default=None
-
when the last-layer features is a tensor of dim >= 3, this tells how to reduce -it into a dim-2 tensor. E.g. in LLMs for non-language modeling problems, -the penultultimate output is a tensor of shape (batch_size, seq_len, embd_dim). -But the last layer maps (batch_size, embd_dim) to (batch_size, n_classes). -Note: Make sure that this option faithfully reflects the reduction in the model -definition. When inputting a string, available options are -{'pick_first', 'pick_last', 'average'}.
-
dict_key_x : str, default='input_ids'
-
The dictionary key under which the input tensor x is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
dict_key_y : str, default='labels'
-
The dictionary key under which the target tensor y is stored. Only has effect -when the model takes a MutableMapping as the input. Useful for Huggingface -LLM models.
-
backend : subclasses of CurvatureInterface
-
backend for access to curvature/Hessian approximations
-
last_layer_name : str, default=None
-
name of the model's last layer, if None it will be determined automatically
-
backend_kwargs : dict, default=None
-
arguments passed to the backend on initialization, for example to -set the number of MC samples for stochastic approximations.
-
-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop prior_precision_diag : torch.Tensor
-
-

Obtain the diagonal prior precision p_0 constructed from either -a scalar or diagonal prior precision.

-

Returns

-
-
prior_precision_diag : torch.Tensor
-
 
-
-
-
-

Methods

-
-
-def fit(self, train_loader: DataLoader, override: bool = True, progress_bar: bool = False) ‑> None -
-
-

Fit the local Laplace approximation at the parameters of the model.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
each iterate is a training batch, either (X, y) tensors or a dict-like -object containing keys as expressed by self.dict_key_x and -self.dict_key_y. train_loader.dataset needs to be set to access -N, size of the data set.
-
override : bool, default=True
-
whether to initialize H, loss, and n_data again; setting to False is useful for -online learning settings to accumulate a sequential posterior approximation.
-
progress_bar : bool, default=False
-
 
-
-
-
-def functional_variance_fast(self, X) -
-
-

Should be overriden if there exists a trick to make this fast!

-

Parameters

-
-
X : torch.Tensor of shape (batch_size, input_dim)
-
 
-
-

Returns

-
-
f_var_diag : torch.Tensor of shape (batch_size, num_outputs)
-
Corresponding to the diagonal of the covariance matrix of the outputs
-
-
-
-def state_dict(self) ‑> dict[str, typing.Any] -
-
-
-
-
-def load_state_dict(self, state_dict: dict[str, Any]) ‑> None -
-
-
-
-
-

Inherited members

- -
-
-class FullLLLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, last_layer_name: str | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Last-layer Laplace approximation with full, i.e., dense, log likelihood Hessian approximation -and hence posterior precision. Based on the chosen backend parameter, the full -approximation can be, for example, a generalized Gauss-Newton matrix. -Mathematically, we have P \in \mathbb{R}^{P \times P}. -See FullLaplace, LLLaplace, and BaseLaplace for the full interface.

-

Ancestors

- -

Inherited members

- -
-
-class KronLLLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, last_layer_name: str | None = None, damping: bool = False, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Last-layer Laplace approximation with Kronecker factored log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have for the last parameter group, i.e., torch.nn.Linear, -that \P\approx Q \otimes H. -See KronLaplace, LLLaplace, and BaseLaplace for the full interface and see -Kron and KronDecomposed for the structure of -the Kronecker factors. Kron is used to aggregate factors by summing up and -KronDecomposed is used to add the prior, a Hessian factor (e.g. temperature), -and computing posterior covariances, marginal likelihood, etc. -Use of damping is possible by initializing or setting damping=True.

-

Ancestors

- -

Inherited members

- -
-
-class DiagLLLaplace -(model: nn.Module, likelihood: Likelihood | str, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', backend: type[CurvatureInterface] | None = None, last_layer_name: str | None = None, backend_kwargs: dict[str, Any] | None = None, asdl_fisher_kwargs: dict[str, Any] | None = None) -
-
-

Last-layer Laplace approximation with diagonal log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have P \approx \textrm{diag}(P). -See DiagLaplace, LLLaplace, and BaseLaplace for the full interface.

-

Ancestors

- -

Inherited members

- -
-
-class FunctionalLLLaplace -(model: nn.Module, likelihood: Likelihood | str, n_subset: int, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None, dict_key_x: str = 'input_ids', dict_key_y: str = 'labels', last_layer_name: str | None = None, backend: type[CurvatureInterface] | None = laplace.curvature.backpack.BackPackGGN, backend_kwargs: dict[str, Any] | None = None, independent_outputs: bool = False, seed: int = 0) -
-
-

Here not much changes in terms of GP inference compared to FunctionalLaplace class. -Since now we treat only the last layer probabilistically and the rest of the network is used as a "fixed feature -extractor", that means that the X \in \mathbb{R}^{M \times D} in GP inference changes -to \tilde{X} \in \mathbb{R}^{M \times l_{n-1}} , -where l_{n-1} is the dimension of the output -of the penultimate NN layer.

-

See FunctionalLaplace for the full interface.

-

Ancestors

- -

Methods

-
-
-def fit(self, train_loader: DataLoader) ‑> None -
-
-

Fit the Laplace approximation of a GP posterior.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
train_loader.dataset needs to be set to access N, size of the data set -train_loader.batch_size needs to be set to access b batch_size
-
-
-
-def state_dict(self) ‑> dict -
-
-
-
-
-def load_state_dict(self, state_dict: dict) -
-
-
-
-
-

Inherited members

- -
-
-
-
- -
- - - diff --git a/examples/regression_example.md b/docs/regression_example.md similarity index 89% rename from examples/regression_example.md rename to docs/regression_example.md index 77dcc2b2..778d7c0f 100644 --- a/examples/regression_example.md +++ b/docs/regression_example.md @@ -1,11 +1,9 @@ -## Full example: Optimization of the marginal likelihood and prediction - -### Sinusoidal toy data +## Sinusoidal toy data We show how the marginal likelihood can be used after training a MAP network on a simple sinusoidal regression task. Subsequently, we use the optimized LA to predict which provides uncertainty on top of the MAP prediction. We also show how the `marglik_training` utility method can be used to jointly train the MAP and hyperparameters. -First, we set up the training data for the problem with observation noise \\(\\sigma=0.3\\): +First, we set up the training data for the problem with observation noise \(\sigma=0.3\): ```python from laplace.baselaplace import FullLaplace @@ -24,7 +22,7 @@ torch.manual_seed(711) X_train, y_train, train_loader, X_test = get_sinusoid_example(sigma_noise=0.3) ``` -### Training a MAP +## Training a MAP We now use `pytorch` to train a neural network with single hidden layer and Tanh activation. The trained neural network will be our MAP estimate. @@ -49,7 +47,7 @@ for i in range(n_epochs): optimizer.step() ``` -### Fitting and optimizing the Laplace approximation using empirical Bayes +## Fitting and optimizing the Laplace approximation using empirical Bayes With the MAP-trained model at hand, we can estimate the prior precision and observation noise using empirical Bayes after training. @@ -70,15 +68,15 @@ for i in range(n_epochs): hyper_optimizer.step() ``` -The obtained observation noise is close to the ground truth with a value of \\(\\sigma \\approx 0.28\\) +The obtained observation noise is close to the ground truth with a value of \(\sigma \approx 0.28\) without the need for any validation data. -The resulting prior precision is \\(\\delta \\approx 0.10\\). +The resulting prior precision is \(\delta \approx 0.10\). -### Bayesian predictive +## Bayesian predictive Here, we compare the MAP prediction to the obtained LA prediction. -For LA, we have a closed-form predictive distribution on the output \\(f\\) which is a Gaussian -\\(\\mathcal{N}(f(x;\\theta\_{MAP}), \\mathbb{V}[f] + \\sigma^2)\\): +For LA, we have a closed-form predictive distribution on the output \(f\) which is a Gaussian +\(\mathcal{N}(f(x;\theta\_{MAP}), \mathbb{V}[f] + \sigma^2)\): ```python x = X_test.flatten().cpu().numpy() @@ -97,7 +95,7 @@ In comparison to the MAP, the predictive shows useful uncertainties. When our MAP is over or underfit, the Laplace approximation cannot fix this anymore. In this case, joint optimization of MAP and marginal likelihood can be useful. -### Jointly optimize MAP and hyperparameters using online empirical Bayes +## Jointly optimize MAP and hyperparameters using online empirical Bayes We provide a utility method `marglik_training` that implements the algorithm proposed in [1]. The method optimizes the neural network and the hyperparameters in an interleaved way diff --git a/examples/reward_modeling_example.md b/docs/reward_modeling_example.md similarity index 97% rename from examples/reward_modeling_example.md rename to docs/reward_modeling_example.md index a338572a..342bc67b 100644 --- a/examples/reward_modeling_example.md +++ b/docs/reward_modeling_example.md @@ -1,9 +1,9 @@ -## Full Example: Bayesian Bradley-Terry Reward Modeling - The `laplace-torch` library can also be used to "Bayesianize" a pretrained Bradley-Terry reward model, popular in large language models. See for a primer in reward modeling. +## Defining a preference dataset + First order of business, let's define our comparison dataset. We will use the `datasets` library from Huggingface to handle the data. @@ -41,6 +41,8 @@ data_dict = [ dataset = Dataset.from_list(data_dict) ``` +## Defining a reward model + Now, let's define the reward model. During training, it assumes that `x` is a tensor of shape `(batch_size, 2, dim)`, which is a concatenation of `x0` and `x1` above. The second dimension of size 2 is preserved through the forward pass, resulting in @@ -91,6 +93,8 @@ class SimpleRewardModel(nn.Module): return logits ``` +## Data preprocessing + To fulfill the 3D tensor requirement, we need to preprocess the dict-based dataset. ```python @@ -108,6 +112,8 @@ tensor_dataloader = data_utils.DataLoader( ) ``` +## MAP training + Then, we can train as usual using the cross entropy loss. ```python @@ -124,6 +130,8 @@ for epoch in range(10): opt.step() ``` +## Applying Laplace + Applying Laplace to this model is a breeze. Simply state that the likelihood is `reward_modeling`. ```python diff --git a/docs/subnetlaplace.html b/docs/subnetlaplace.html deleted file mode 100644 index bb11f14e..00000000 --- a/docs/subnetlaplace.html +++ /dev/null @@ -1,291 +0,0 @@ - - - - - - -laplace.subnetlaplace API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.subnetlaplace

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class SubnetLaplace -(model: nn.Module, likelihood: Likelihood | str, subnetwork_indices: torch.LongTensor, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, backend: Type[CurvatureInterface] | None = None, backend_kwargs: dict | None = None, asdl_fisher_kwargs: dict | None = None) -
-
-

Class for subnetwork Laplace, which computes the Laplace approximation over just a subset -of the model parameters (i.e. a subnetwork within the neural network), as proposed in [1]. -Subnetwork Laplace can only be used with either a full or a diagonal Hessian approximation.

-

A Laplace approximation is represented by a MAP which is given by the -model parameter and a posterior precision or covariance specifying -a Gaussian distribution \mathcal{N}(\theta_{MAP}, P^{-1}). -Here, only a subset of the model parameters (i.e. a subnetwork of the -neural network) are treated probabilistically. -The goal of this class is to compute the posterior precision P -which sums as - -P = \sum_{n=1}^N \nabla^2_\theta \log p(\mathcal{D}_n \mid \theta) -\vert_{\theta_{MAP}} + \nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}}. - -The prior is assumed to be Gaussian and therefore we have a simple form for -\nabla^2_\theta \log p(\theta) \vert_{\theta_{MAP}} = P_0 . -In particular, we assume a scalar or diagonal prior precision so that in -all cases P_0 = \textrm{diag}(p_0) and the structure of p_0 can be varied.

-

The subnetwork Laplace approximation only supports a full, i.e., dense, log likelihood -Hessian approximation and hence posterior precision. -Based on the chosen backend -parameter, the full approximation can be, for example, a generalized Gauss-Newton -matrix. -Mathematically, we have P \in \mathbb{R}^{P \times P}. -See FullLaplace and BaseLaplace for the full interface.

-

References

-

[1] Daxberger, E., Nalisnick, E., Allingham, JU., Antorán, J., Hernández-Lobato, JM. -Bayesian Deep Learning via Subnetwork Inference. -ICML 2021.

-

Parameters

-
-
model : torch.nn.Module or FeatureExtractor
-
 
-
likelihood : {'classification', 'regression'}
-
determines the log likelihood Hessian approximation
-
subnetwork_indices : torch.LongTensor
-
indices of the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters())) -that define the subnetwork to apply the Laplace approximation over
-
sigma_noise : torch.Tensor or float, default=1
-
observation noise for the regression setting; must be 1 for classification
-
prior_precision : torch.Tensor or float, default=1
-
prior precision of a Gaussian prior (= weight decay); -can be scalar, per-layer, or diagonal in the most general case
-
prior_mean : torch.Tensor or float, default=0
-
prior mean of a Gaussian prior, useful for continual learning
-
temperature : float, default=1
-
temperature of the likelihood; lower temperature leads to more -concentrated posterior and vice versa.
-
backend : subclasses of CurvatureInterface
-
backend for access to curvature/Hessian approximations
-
backend_kwargs : dict, default=None
-
arguments passed to the backend on initialization, for example to -set the number of MC samples for stochastic approximations.
-
-

Ancestors

- -

Subclasses

- -

Instance variables

-
-
prop prior_precision_diag : torch.Tensor
-
-

Obtain the diagonal prior precision p_0 constructed from either -a scalar or diagonal prior precision.

-

Returns

-
-
prior_precision_diag : torch.Tensor
-
 
-
-
-
prop mean_subnet : torch.Tensor
-
-
-
-
-

Methods

-
-
-def assemble_full_samples(self, subnet_samples) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class FullSubnetLaplace -(model: nn.Module, likelihood: Likelihood | str, subnetwork_indices: torch.LongTensor, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, backend: Type[CurvatureInterface] | None = None, backend_kwargs: dict | None = None, asdl_fisher_kwargs: dict | None = None) -
-
-

Subnetwork Laplace approximation with full, i.e., dense, log likelihood Hessian -approximation and hence posterior precision. Based on the chosen backend parameter, -the full approximation can be, for example, a generalized Gauss-Newton matrix. -Mathematically, we have P \in \mathbb{R}^{P \times P}. -See FullLaplace, SubnetLaplace, and BaseLaplace for the full interface.

-

Ancestors

- -

Inherited members

- -
-
-class DiagSubnetLaplace -(model: nn.Module, likelihood: Likelihood | str, subnetwork_indices: torch.LongTensor, sigma_noise: float | torch.Tensor = 1.0, prior_precision: float | torch.Tensor = 1.0, prior_mean: float | torch.Tensor = 0.0, temperature: float = 1.0, backend: Type[CurvatureInterface] | None = None, backend_kwargs: dict | None = None, asdl_fisher_kwargs: dict | None = None) -
-
-

Subnetwork Laplace approximation with diagonal log likelihood Hessian approximation -and hence posterior precision. -Mathematically, we have P \approx \textrm{diag}(P). -See DiagLaplace, SubnetLaplace, and BaseLaplace for the full interface.

-

Ancestors

- -

Inherited members

- -
-
-
-
- -
- - - diff --git a/docs/utils/enums.html b/docs/utils/enums.html deleted file mode 100644 index e7da94bd..00000000 --- a/docs/utils/enums.html +++ /dev/null @@ -1,282 +0,0 @@ - - - - - - -laplace.utils.enums API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.utils.enums

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class SubsetOfWeights -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var ALL
-
-
-
-
var LAST_LAYER
-
-
-
-
var SUBNETWORK
-
-
-
-
-
-
-class HessianStructure -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var FULL
-
-
-
-
var KRON
-
-
-
-
var DIAG
-
-
-
-
var LOWRANK
-
-
-
-
var GP
-
-
-
-
-
-
-class Likelihood -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var REGRESSION
-
-
-
-
var CLASSIFICATION
-
-
-
-
var REWARD_MODELING
-
-
-
-
-
-
-class PredType -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var GLM
-
-
-
-
var NN
-
-
-
-
var GP
-
-
-
-
-
-
-class LinkApprox -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var MC
-
-
-
-
var PROBIT
-
-
-
-
var BRIDGE
-
-
-
-
var BRIDGE_NORM
-
-
-
-
-
-
-class TuningMethod -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var MARGLIK
-
-
-
-
var GRIDSEARCH
-
-
-
-
-
-
-class PriorStructure -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var SCALAR
-
-
-
-
var DIAG
-
-
-
-
var LAYERWISE
-
-
-
-
-
-
-
-
- -
- - - diff --git a/docs/utils/feature_extractor.html b/docs/utils/feature_extractor.html deleted file mode 100644 index d68359de..00000000 --- a/docs/utils/feature_extractor.html +++ /dev/null @@ -1,212 +0,0 @@ - - - - - - -laplace.utils.feature_extractor API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.utils.feature_extractor

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class FeatureReduction -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

Possible choices of feature reduction before applying last-layer Laplace.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var PICK_FIRST
-
-
-
-
var PICK_LAST
-
-
-
-
var AVERAGE
-
-
-
-
-
-
-class FeatureExtractor -(model: nn.Module, last_layer_name: str | None = None, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None) -
-
-

Feature extractor for a PyTorch neural network. -A wrapper which can return the output of the penultimate layer in addition to -the output of the last layer for each forward pass. If the name of the last -layer is not known, it can determine it automatically. It assumes that the -last layer is linear and that for every forward pass the last layer is the same. -If the name of the last layer is known, it can be passed as a parameter at -initilization; this is the safest way to use this class. -Based on https://gist.github.com/fkodom/27ed045c9051a39102e8bcf4ce31df76.

-

Parameters

-
-
model : torch.nn.Module
-
PyTorch model
-
last_layer_name : str, default=None
-
if the name of the last layer is already known, otherwise it will -be determined automatically.
-
enable_backprop : bool, default=False
-
whether to enable backprop through the feature extactor to get the gradients of -the inputs. Useful for e.g. Bayesian optimization.
-
feature_reduction : FeatureReduction or str, default=None
-
when the last-layer features is a tensor of dim >= 3, this tells how to reduce -it into a dim-2 tensor. E.g. in LLMs for non-language modeling problems, -the penultultimate output is a tensor of shape (batch_size, seq_len, embd_dim). -But the last layer maps (batch_size, embd_dim) to (batch_size, n_classes). -Note: Make sure that this option faithfully reflects the reduction in the model -definition. When inputting a string, available options are -{'pick_first', 'pick_last', 'average'}.
-
-

Initialize internal Module state, shared by both nn.Module and ScriptModule.

-

Ancestors

-
    -
  • torch.nn.modules.module.Module
    • -
-

Class variables

-
-
var dump_patches : bool
-
-
-
-
var training : bool
-
-
-
-
var call_super_init : bool
-
-
-
-
-

Methods

-
-
-def forward(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any]) ‑> Callable[..., Any] -
-
-

Forward pass. If the last layer is not known yet, it will be -determined when this function is called for the first time.

-

Parameters

-
-
x : torch.Tensor or a dict-like object containing the input tensors
-
one batch of data to use as input for the forward pass
-
-
-
-def forward_with_features(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any]) -
-
-

Forward pass which returns the output of the penultimate layer along -with the output of the last layer. If the last layer is not known yet, -it will be determined when this function is called for the first time.

-

Parameters

-
-
x : torch.Tensor or a dict-like object containing the input tensors
-
one batch of data to use as input for the forward pass
-
-
-
-def set_last_layer(self, last_layer_name: str) ‑> None -
-
-

Set the last layer of the model by its name. This sets the forward -hook to get the output of the penultimate layer.

-

Parameters

-
-
last_layer_name : str
-
the name of the last layer (fixed in model.named_modules()).
-
-
-
-def find_last_layer(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any]) -
-
-

Automatically determines the last layer of the model with one -forward pass. It assumes that the last layer is the same for every -forward pass and that it is an instance of torch.nn.Linear. -Might not work with every architecture, but is tested with all PyTorch -torchvision classification models (besides SqueezeNet, which has no -linear last layer).

-

Parameters

-
-
x : torch.Tensor or dict-like object containing the input tensors
-
one batch of data to use as input for the forward pass
-
-
-
-
-
-
-
- -
- - - diff --git a/docs/utils/index.html b/docs/utils/index.html deleted file mode 100644 index d742b699..00000000 --- a/docs/utils/index.html +++ /dev/null @@ -1,1473 +0,0 @@ - - - - - - -laplace.utils API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.utils

-
-
-
-
-

Sub-modules

-
-
laplace.utils.enums
-
-
-
-
laplace.utils.feature_extractor
-
-
-
-
laplace.utils.matrix
-
-
-
-
laplace.utils.metrics
-
-
-
-
laplace.utils.subnetmask
-
-
-
-
laplace.utils.swag
-
-
-
-
laplace.utils.utils
-
-
-
-
-
-
-
-
-

Functions

-
-
-def get_nll(out_dist: torch.Tensor, targets: torch.Tensor) ‑> torch.Tensor -
-
-
-
-
-def validate(laplace: BaseLaplace, val_loader: DataLoader, loss: torchmetrics.Metric | Callable[[torch.Tensor, torch.Tensor], torch.Tensor] | Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor], pred_type: PredType | str = PredType.GLM, link_approx: LinkApprox | str = LinkApprox.PROBIT, n_samples: int = 100, dict_key_y: str = 'labels') -
-
-
-
-
-def parameters_per_layer(model: nn.Module) ‑> list[int] -
-
-

Get number of parameters per layer.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
-

Returns

-
-
params_per_layer : list[int]
-
 
-
-
-
-def invsqrt_precision(M: torch.Tensor) ‑> torch.Tensor -
-
-

Compute M^{-0.5} as a tridiagonal matrix.

-

Parameters

-
-
M : torch.Tensor
-
 
-
-

Returns

-
-
M_invsqrt : torch.Tensor
-
 
-
-
-
-def kron(t1: torch.Tensor, t2: torch.Tensor) ‑> torch.Tensor -
-
-

Computes the Kronecker product between two tensors.

-

Parameters

-
-
t1 : torch.Tensor
-
 
-
t2 : torch.Tensor
-
 
-
-

Returns

-
-
kron_product : torch.Tensor
-
 
-
-
-
-def diagonal_add_scalar(X: torch.Tensor, value: torch.Tensor) ‑> torch.Tensor -
-
-

Add scalar value value to diagonal of X.

-

Parameters

-
-
X : torch.Tensor
-
 
-
value : torch.Tensor or float
-
 
-
-

Returns

-
-
X_add_scalar : torch.Tensor
-
 
-
-
-
-def symeig(M: torch.Tensor) ‑> tuple[torch.Tensor, torch.Tensor] -
-
-

Symetric eigendecomposition avoiding failure cases by -adding and removing jitter to the diagonal.

-

Parameters

-
-
M : torch.Tensor
-
 
-
-

Returns

-
-
L : torch.Tensor
-
eigenvalues
-
W : torch.Tensor
-
eigenvectors
-
-
-
-def block_diag(blocks: list[torch.Tensor]) ‑> torch.Tensor -
-
-

Compose block-diagonal matrix of individual blocks.

-

Parameters

-
-
blocks : list[torch.Tensor]
-
 
-
-

Returns

-
-
M : torch.Tensor
-
 
-
-
-
-def normal_samples(mean: torch.Tensor, var: torch.Tensor, n_samples: int, generator: torch.Generator | None = None) -
-
-

Produce samples from a batch of Normal distributions either parameterized -by a diagonal or full covariance given by var.

-

Parameters

-
-
mean : torch.Tensor
-
(batch_size, output_dim)
-
var : torch.Tensor
-
(co)variance of the Normal distribution -(batch_size, output_dim, output_dim) or (batch_size, output_dim)
-
generator : torch.Generator
-
random number generator
-
-
-
-def _is_batchnorm(module: nn.Module) ‑> bool -
-
-
-
-
-def _is_valid_scalar(scalar: float | int | torch.Tensor) -
-
-
-
-
-def expand_prior_precision(prior_prec: torch.Tensor, model: nn.Module) ‑> torch.Tensor -
-
-

Expand prior precision to match the shape of the model parameters.

-

Parameters

-
-
prior_prec : torch.Tensor 1-dimensional
-
prior precision
-
model : torch.nn.Module
-
torch model with parameters that are regularized by prior_prec
-
-

Returns

-
-
expanded_prior_prec : torch.Tensor
-
expanded prior precision has the same shape as model parameters
-
-
-
-def fix_prior_prec_structure(prior_prec_init: torch.Tensor, prior_structure: PriorStructure | str, n_layers: int, n_params: int, device: torch.device) -
-
-

Create a tensor of prior precision with the correct shape, depending on the -choice of the prior structure type.

-

Parameters

-
-
prior_prec_init : torch.Tensor
-
the initial prior precision tensor (could be scalar)
-
prior_structure : PriorStructure | str
-
the choice of the prior structure type
-
n_layers : int
-
 
-
n_params : int
-
 
-
device : torch.device
-
 
-
-

Returns

-
-
correct_prior_precision : torch.Tensor
-
 
-
-
-
-def fit_diagonal_swag_var(model: nn.Module, train_loader: DataLoader, criterion: nn.CrossEntropyLoss | nn.MSELoss, n_snapshots_total: int = 40, snapshot_freq: int = 1, lr: float = 0.01, momentum: float = 0.9, weight_decay: float = 0.0003, min_var: float = 1e-30) -
-
-

Fit diagonal SWAG [1], which estimates marginal variances of model parameters by -computing the first and second moment of SGD iterates with a large learning rate.

-

Implementation partly adapted from: -- https://github.com/wjmaddox/swa_gaussian/blob/master/swag/posteriors/swag.py -- https://github.com/wjmaddox/swa_gaussian/blob/master/experiments/train/run_swag.py

-

References

-

[1] Maddox, W., Garipov, T., Izmailov, P., Vetrov, D., Wilson, AG. -A Simple Baseline for Bayesian Uncertainty in Deep Learning. -NeurIPS 2019.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
train_loader : torch.data.utils.DataLoader
-
training data loader to use for snapshot collection
-
criterion : torch.nn.CrossEntropyLoss or torch.nn.MSELoss
-
loss function to use for snapshot collection
-
n_snapshots_total : int
-
total number of model snapshots to collect
-
snapshot_freq : int
-
snapshot collection frequency (in epochs)
-
lr : float
-
SGD learning rate for collecting snapshots
-
momentum : float
-
SGD momentum
-
weight_decay : float
-
SGD weight decay
-
min_var : float
-
minimum parameter variance to clamp to (for numerical stability)
-
-

Returns

-
-
param_variances : torch.Tensor
-
vector of marginal variances for each model parameter
-
-
-
-
-
-

Classes

-
-
-class SoDSampler -(N, M, seed: int = 0) -
-
-

Base class for all Samplers.

-

Every Sampler subclass has to provide an :meth:__iter__ method, providing a -way to iterate over indices or lists of indices (batches) of dataset elements, -and may provide a :meth:__len__ method that returns the length of the returned iterators.

-

Args

-
-
data_source : Dataset
-
This argument is not used and will be removed in 2.2.0. -You may still have custom implementation that utilizes it.
-
-

Example

-
>>> # xdoctest: +SKIP
->>> class AccedingSequenceLengthSampler(Sampler[int]):
->>>     def __init__(self, data: List[str]) -> None:
->>>         self.data = data
->>>
->>>     def __len__(self) -> int:
->>>         return len(self.data)
->>>
->>>     def __iter__(self) -> Iterator[int]:
->>>         sizes = torch.tensor([len(x) for x in self.data])
->>>         yield from torch.argsort(sizes).tolist()
->>>
->>> class AccedingSequenceLengthBatchSampler(Sampler[List[int]]):
->>>     def __init__(self, data: List[str], batch_size: int) -> None:
->>>         self.data = data
->>>         self.batch_size = batch_size
->>>
->>>     def __len__(self) -> int:
->>>         return (len(self.data) + self.batch_size - 1) // self.batch_size
->>>
->>>     def __iter__(self) -> Iterator[List[int]]:
->>>         sizes = torch.tensor([len(x) for x in self.data])
->>>         for batch in torch.chunk(torch.argsort(sizes), len(self)):
->>>             yield batch.tolist()
-
-
-

Note: The :meth:__len__ method isn't strictly required by

-

:class:~torch.utils.data.DataLoader, but is expected in any -calculation involving the length of a :class:~torch.utils.data.DataLoader.

-
-

Ancestors

-
    -
  • torch.utils.data.sampler.Sampler
    • -
  • typing.Generic
    • -
-
-
-class FeatureExtractor -(model: nn.Module, last_layer_name: str | None = None, enable_backprop: bool = False, feature_reduction: FeatureReduction | str | None = None) -
-
-

Feature extractor for a PyTorch neural network. -A wrapper which can return the output of the penultimate layer in addition to -the output of the last layer for each forward pass. If the name of the last -layer is not known, it can determine it automatically. It assumes that the -last layer is linear and that for every forward pass the last layer is the same. -If the name of the last layer is known, it can be passed as a parameter at -initilization; this is the safest way to use this class. -Based on https://gist.github.com/fkodom/27ed045c9051a39102e8bcf4ce31df76.

-

Parameters

-
-
model : torch.nn.Module
-
PyTorch model
-
last_layer_name : str, default=None
-
if the name of the last layer is already known, otherwise it will -be determined automatically.
-
enable_backprop : bool, default=False
-
whether to enable backprop through the feature extactor to get the gradients of -the inputs. Useful for e.g. Bayesian optimization.
-
feature_reduction : FeatureReduction or str, default=None
-
when the last-layer features is a tensor of dim >= 3, this tells how to reduce -it into a dim-2 tensor. E.g. in LLMs for non-language modeling problems, -the penultultimate output is a tensor of shape (batch_size, seq_len, embd_dim). -But the last layer maps (batch_size, embd_dim) to (batch_size, n_classes). -Note: Make sure that this option faithfully reflects the reduction in the model -definition. When inputting a string, available options are -{'pick_first', 'pick_last', 'average'}.
-
-

Initialize internal Module state, shared by both nn.Module and ScriptModule.

-

Ancestors

-
    -
  • torch.nn.modules.module.Module
    • -
-

Class variables

-
-
var dump_patches : bool
-
-
-
-
var training : bool
-
-
-
-
var call_super_init : bool
-
-
-
-
-

Methods

-
-
-def forward(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any]) ‑> Callable[..., Any] -
-
-

Forward pass. If the last layer is not known yet, it will be -determined when this function is called for the first time.

-

Parameters

-
-
x : torch.Tensor or a dict-like object containing the input tensors
-
one batch of data to use as input for the forward pass
-
-
-
-def forward_with_features(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any]) -
-
-

Forward pass which returns the output of the penultimate layer along -with the output of the last layer. If the last layer is not known yet, -it will be determined when this function is called for the first time.

-

Parameters

-
-
x : torch.Tensor or a dict-like object containing the input tensors
-
one batch of data to use as input for the forward pass
-
-
-
-def set_last_layer(self, last_layer_name: str) ‑> None -
-
-

Set the last layer of the model by its name. This sets the forward -hook to get the output of the penultimate layer.

-

Parameters

-
-
last_layer_name : str
-
the name of the last layer (fixed in model.named_modules()).
-
-
-
-def find_last_layer(self, x: torch.Tensor | MutableMapping[str, torch.Tensor | Any]) -
-
-

Automatically determines the last layer of the model with one -forward pass. It assumes that the last layer is the same for every -forward pass and that it is an instance of torch.nn.Linear. -Might not work with every architecture, but is tested with all PyTorch -torchvision classification models (besides SqueezeNet, which has no -linear last layer).

-

Parameters

-
-
x : torch.Tensor or dict-like object containing the input tensors
-
one batch of data to use as input for the forward pass
-
-
-
-
-
-class Kron -(kfacs: list[tuple[torch.Tensor] | torch.Tensor]) -
-
-

Kronecker factored approximate curvature representation for a corresponding -neural network. -Each element in kfacs is either a tuple or single matrix. -A tuple represents two Kronecker factors Q, and H and a single element -is just a full block Hessian approximation.

-

Parameters

-
-
kfacs : list[Iterable[torch.Tensor] | torch.Tensor]
-
each element in the list is a tuple of two Kronecker factors Q, H -or a single matrix approximating the Hessian (in case of bias, for example)
-
-

Static methods

-
-
-def init_from_model(model: nn.Module | Iterable[nn.Parameter], device: torch.device) -
-
-

Initialize Kronecker factors based on a models architecture.

-

Parameters

-
-
model : nn.Module or iterable of parameters, e.g. model.parameters()
-
 
-
device : torch.device
-
 
-
-

Returns

-
-
kron : Kron
-
 
-
-
-
-

Methods

-
-
-def decompose(self, damping: bool = False) ‑> KronDecomposed -
-
-

Eigendecompose Kronecker factors and turn into KronDecomposed. -Parameters

-
-
-
damping : bool
-
use damping
-
-

Returns

-
-
kron_decomposed : KronDecomposed
-
 
-
-
-
-def bmm(self, W: torch.Tensor, exponent: float = 1) ‑> torch.Tensor -
-
-

Batched matrix multiplication with the Kronecker factors. -If Kron is H, we compute H @ W. -This is useful for computing the predictive or a regularization -based on Kronecker factors as in continual learning.

-

Parameters

-
-
W : torch.Tensor
-
matrix (batch, classes, params)
-
exponent : float, default=1
-
only can be 1 for Kron, requires KronDecomposed for other -exponent values of the Kronecker factors.
-
-

Returns

-
-
SW : torch.Tensor
-
result (batch, classes, params)
-
-
-
-def logdet(self) ‑> torch.Tensor -
-
-

Compute log determinant of the Kronecker factors and sums them up. -This corresponds to the log determinant of the entire Hessian approximation.

-

Returns

-
-
logdet : torch.Tensor
-
 
-
-
-
-def diag(self) ‑> torch.Tensor -
-
-

Extract diagonal of the entire Kronecker factorization.

-

Returns

-
-
diag : torch.Tensor
-
 
-
-
-
-def to_matrix(self) ‑> torch.Tensor -
-
-

Make the Kronecker factorization dense by computing the kronecker product. -Warning: this should only be used for testing purposes as it will allocate -large amounts of memory for big architectures.

-

Returns

-
-
block_diag : torch.Tensor
-
 
-
-
-
-
-
-class KronDecomposed -(eigenvectors: list[tuple[torch.Tensor]], eigenvalues: list[tuple[torch.Tensor]], deltas: torch.Tensor | None = None, damping: bool = False) -
-
-

Decomposed Kronecker factored approximate curvature representation -for a corresponding neural network. -Each matrix in Kron is decomposed to obtain KronDecomposed. -Front-loading decomposition allows cheap repeated computation -of inverses and log determinants. -In contrast to Kron, we can add scalar or layerwise scalars but -we cannot add other Kron or KronDecomposed anymore.

-

Parameters

-
-
eigenvectors : list[Tuple[torch.Tensor]]
-
eigenvectors corresponding to matrices in a corresponding Kron
-
eigenvalues : list[Tuple[torch.Tensor]]
-
eigenvalues corresponding to matrices in a corresponding Kron
-
deltas : torch.Tensor
-
addend for each group of Kronecker factors representing, for example, -a prior precision
-
dampen : bool, default=False
-
use dampen approximation mixing prior and Kron partially multiplicatively
-
-

Methods

-
-
-def detach(self) ‑> KronDecomposed -
-
-
-
-
-def logdet(self) ‑> torch.Tensor -
-
-

Compute log determinant of the Kronecker factors and sums them up. -This corresponds to the log determinant of the entire Hessian approximation. -In contrast to Kron.logdet(), additive deltas corresponding to prior -precisions are added.

-

Returns

-
-
logdet : torch.Tensor
-
 
-
-
-
-def inv_square_form(self, W: torch.Tensor) ‑> torch.Tensor -
-
-
-
-
-def bmm(self, W: torch.Tensor, exponent: float = -1) ‑> torch.Tensor -
-
-

Batched matrix multiplication with the decomposed Kronecker factors. -This is useful for computing the predictive or a regularization loss. -Compared to Kron.bmm(), a prior can be added here in form of deltas -and the exponent can be other than just 1. -Computes H^{exponent} W.

-

Parameters

-
-
W : torch.Tensor
-
matrix (batch, classes, params)
-
exponent : float, default=1
-
 
-
-

Returns

-
-
SW : torch.Tensor
-
result (batch, classes, params)
-
-
-
-def diag(self, exponent: float = 1) ‑> torch.Tensor -
-
-

Extract diagonal of the entire decomposed Kronecker factorization.

-

Parameters

-
-
exponent : float, default=1
-
exponent of the Kronecker factorization
-
-

Returns

-
-
diag : torch.Tensor
-
 
-
-
-
-def to_matrix(self, exponent: float = 1) ‑> torch.Tensor -
-
-

Make the Kronecker factorization dense by computing the kronecker product. -Warning: this should only be used for testing purposes as it will allocate -large amounts of memory for big architectures.

-

Parameters

-
-
exponent : float, default=1
-
exponent of the Kronecker factorization
-
-

Returns

-
-
block_diag : torch.Tensor
-
 
-
-
-
-
-
-class SubnetMask -(model: nn.Module) -
-
-

Baseclass for all subnetwork masks in this library (for subnetwork Laplace).

-

Parameters

-
-
model : torch.nn.Module
-
 
-
-

Subclasses

- -

Instance variables

-
-
prop indices : torch.LongTensor
-
-
-
-
prop n_params_subnet : int
-
-
-
-
-

Methods

-
-
-def convert_subnet_mask_to_indices(self, subnet_mask: torch.Tensor) ‑> torch.LongTensor -
-
-

Converts a subnetwork mask into subnetwork indices.

-

Parameters

-
-
subnet_mask : torch.Tensor
-
a binary vector of size (n_params) where 1s locate the subnetwork parameters -within the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters()))
-
-

Returns

-
-
subnet_mask_indices : torch.LongTensor
-
a vector of indices of the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters())) -that define the subnetwork
-
-
-
-def select(self, train_loader: DataLoader | None = None) -
-
-

Select the subnetwork mask.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader, default=None
-
each iterate is a training batch (X, y); -train_loader.dataset needs to be set to access N, size of the data set
-
-

Returns

-
-
subnet_mask_indices : torch.LongTensor
-
a vector of indices of the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters())) -that define the subnetwork
-
-
-
-def get_subnet_mask(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-

Get the subnetwork mask.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
each iterate is a training batch (X, y); -train_loader.dataset needs to be set to access N, size of the data set
-
-

Returns

-
-
subnet_mask : torch.Tensor
-
a binary vector of size (n_params) where 1s locate the subnetwork parameters -within the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters()))
-
-
-
-
-
-class RandomSubnetMask -(model: nn.Module, n_params_subnet: int) -
-
-

Subnetwork mask of parameters sampled uniformly at random.

-

Ancestors

-
    -
  • laplace.utils.subnetmask.ScoreBasedSubnetMask
    • -
  • SubnetMask
    • -
-

Methods

-
-
-def compute_param_scores(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class LargestMagnitudeSubnetMask -(model: nn.Module, n_params_subnet: int) -
-
-

Subnetwork mask identifying the parameters with the largest magnitude.

-

Ancestors

-
    -
  • laplace.utils.subnetmask.ScoreBasedSubnetMask
    • -
  • SubnetMask
    • -
-

Methods

-
-
-def compute_param_scores(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class LargestVarianceDiagLaplaceSubnetMask -(model: nn.Module, n_params_subnet: int, diag_laplace_model: DiagLaplace) -
-
-

Subnetwork mask identifying the parameters with the largest marginal variances -(estimated using a diagonal Laplace approximation over all model parameters).

-

Parameters

-
-
model : torch.nn.Module
-
 
-
n_params_subnet : int
-
number of parameters in the subnetwork (i.e. number of top-scoring parameters to select)
-
diag_laplace_model : DiagLaplace
-
diagonal Laplace model to use for variance estimation
-
-

Ancestors

-
    -
  • laplace.utils.subnetmask.ScoreBasedSubnetMask
    • -
  • SubnetMask
    • -
-

Methods

-
-
-def compute_param_scores(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class LargestVarianceSWAGSubnetMask -(model: nn.Module, n_params_subnet: int, likelihood: Likelihood | str = Likelihood.CLASSIFICATION, swag_n_snapshots: int = 40, swag_snapshot_freq: int = 1, swag_lr: float = 0.01) -
-
-

Subnetwork mask identifying the parameters with the largest marginal variances -(estimated using diagonal SWAG over all model parameters).

-

Parameters

-
-
model : torch.nn.Module
-
 
-
n_params_subnet : int
-
number of parameters in the subnetwork (i.e. number of top-scoring parameters to select)
-
likelihood : str
-
'classification' or 'regression'
-
swag_n_snapshots : int
-
number of model snapshots to collect for SWAG
-
swag_snapshot_freq : int
-
SWAG snapshot collection frequency (in epochs)
-
swag_lr : float
-
learning rate for SWAG snapshot collection
-
-

Ancestors

-
    -
  • laplace.utils.subnetmask.ScoreBasedSubnetMask
    • -
  • SubnetMask
    • -
-

Methods

-
-
-def compute_param_scores(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class ParamNameSubnetMask -(model: nn.Module, parameter_names: list[str]) -
-
-

Subnetwork mask corresponding to the specified parameters of the neural network.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
parameter_names : List[str]
-
list of names of the parameters (as in model.named_parameters()) -that define the subnetwork
-
-

Ancestors

- -

Methods

-
-
-def get_subnet_mask(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-

Get the subnetwork mask identifying the specified parameters.

-
-
-

Inherited members

- -
-
-class ModuleNameSubnetMask -(model: nn.Module, module_names: list[str]) -
-
-

Subnetwork mask corresponding to the specified modules of the neural network.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
parameter_names : List[str]
-
list of names of the modules (as in model.named_modules()) that define the subnetwork; -the modules cannot have children, i.e. need to be leaf modules
-
-

Ancestors

- -

Subclasses

- -

Methods

-
-
-def get_subnet_mask(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-

Get the subnetwork mask identifying the specified modules.

-
-
-

Inherited members

- -
-
-class LastLayerSubnetMask -(model: nn.Module, last_layer_name: str | None = None) -
-
-

Subnetwork mask corresponding to the last layer of the neural network.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
last_layer_name : str, default=None
-
name of the model's last layer, if None it will be determined automatically
-
-

Ancestors

- -

Methods

-
-
-def get_subnet_mask(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-

Get the subnetwork mask identifying the last layer.

-
-
-

Inherited members

- -
-
-class RunningNLLMetric -(ignore_index: int = -100) -
-
-

NLL metrics that

-

Parameters

-
-
ignore_index : int, default = -100
-
which class label to ignore when computing the NLL loss
-
-

Initialize internal Module state, shared by both nn.Module and ScriptModule.

-

Ancestors

-
    -
  • torchmetrics.metric.Metric
    • -
  • torch.nn.modules.module.Module
    • -
  • abc.ABC
    • -
-

Class variables

-
-
var is_differentiable : Optional[bool]
-
-
-
-
var higher_is_better : Optional[bool]
-
-
-
-
var full_state_update : Optional[bool]
-
-
-
-
var plot_lower_bound : Optional[float]
-
-
-
-
var plot_upper_bound : Optional[float]
-
-
-
-
var plot_legend_name : Optional[str]
-
-
-
-
-

Methods

-
-
-def update(self, probs: torch.Tensor, targets: torch.Tensor) ‑> None -
-
-

Parameters

-
-
probs : torch.Tensor
-
probability tensor of shape (…, n_classes)
-
targets : torch.Tensor
-
integer tensor of shape (…)
-
-
-
-def compute(self) ‑> torch.Tensor -
-
-

Override this method to compute the final metric value.

-

This method will automatically synchronize state variables when running in distributed backend.

-
-
-
-
-class SubsetOfWeights -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var ALL
-
-
-
-
var LAST_LAYER
-
-
-
-
var SUBNETWORK
-
-
-
-
-
-
-class HessianStructure -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var FULL
-
-
-
-
var KRON
-
-
-
-
var DIAG
-
-
-
-
var LOWRANK
-
-
-
-
var GP
-
-
-
-
-
-
-class Likelihood -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var REGRESSION
-
-
-
-
var CLASSIFICATION
-
-
-
-
var REWARD_MODELING
-
-
-
-
-
-
-class PredType -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var GLM
-
-
-
-
var NN
-
-
-
-
var GP
-
-
-
-
-
-
-class LinkApprox -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var MC
-
-
-
-
var PROBIT
-
-
-
-
var BRIDGE
-
-
-
-
var BRIDGE_NORM
-
-
-
-
-
-
-class TuningMethod -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var MARGLIK
-
-
-
-
var GRIDSEARCH
-
-
-
-
-
-
-class PriorStructure -(value, names=None, *, module=None, qualname=None, type=None, start=1) -
-
-

An enumeration.

-

Ancestors

-
    -
  • builtins.str
    • -
  • enum.Enum
    • -
-

Class variables

-
-
var SCALAR
-
-
-
-
var DIAG
-
-
-
-
var LAYERWISE
-
-
-
-
-
-
-
-
- -
- - - diff --git a/docs/utils/matrix.html b/docs/utils/matrix.html deleted file mode 100644 index 1b66b9ee..00000000 --- a/docs/utils/matrix.html +++ /dev/null @@ -1,312 +0,0 @@ - - - - - - -laplace.utils.matrix API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.utils.matrix

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class Kron -(kfacs: list[tuple[torch.Tensor] | torch.Tensor]) -
-
-

Kronecker factored approximate curvature representation for a corresponding -neural network. -Each element in kfacs is either a tuple or single matrix. -A tuple represents two Kronecker factors Q, and H and a single element -is just a full block Hessian approximation.

-

Parameters

-
-
kfacs : list[Iterable[torch.Tensor] | torch.Tensor]
-
each element in the list is a tuple of two Kronecker factors Q, H -or a single matrix approximating the Hessian (in case of bias, for example)
-
-

Static methods

-
-
-def init_from_model(model: nn.Module | Iterable[nn.Parameter], device: torch.device) -
-
-

Initialize Kronecker factors based on a models architecture.

-

Parameters

-
-
model : nn.Module or iterable of parameters, e.g. model.parameters()
-
 
-
device : torch.device
-
 
-
-

Returns

-
-
kron : Kron
-
 
-
-
-
-

Methods

-
-
-def decompose(self, damping: bool = False) ‑> KronDecomposed -
-
-

Eigendecompose Kronecker factors and turn into KronDecomposed. -Parameters

-
-
-
damping : bool
-
use damping
-
-

Returns

-
-
kron_decomposed : KronDecomposed
-
 
-
-
-
-def bmm(self, W: torch.Tensor, exponent: float = 1) ‑> torch.Tensor -
-
-

Batched matrix multiplication with the Kronecker factors. -If Kron is H, we compute H @ W. -This is useful for computing the predictive or a regularization -based on Kronecker factors as in continual learning.

-

Parameters

-
-
W : torch.Tensor
-
matrix (batch, classes, params)
-
exponent : float, default=1
-
only can be 1 for Kron, requires KronDecomposed for other -exponent values of the Kronecker factors.
-
-

Returns

-
-
SW : torch.Tensor
-
result (batch, classes, params)
-
-
-
-def logdet(self) ‑> torch.Tensor -
-
-

Compute log determinant of the Kronecker factors and sums them up. -This corresponds to the log determinant of the entire Hessian approximation.

-

Returns

-
-
logdet : torch.Tensor
-
 
-
-
-
-def diag(self) ‑> torch.Tensor -
-
-

Extract diagonal of the entire Kronecker factorization.

-

Returns

-
-
diag : torch.Tensor
-
 
-
-
-
-def to_matrix(self) ‑> torch.Tensor -
-
-

Make the Kronecker factorization dense by computing the kronecker product. -Warning: this should only be used for testing purposes as it will allocate -large amounts of memory for big architectures.

-

Returns

-
-
block_diag : torch.Tensor
-
 
-
-
-
-
-
-class KronDecomposed -(eigenvectors: list[tuple[torch.Tensor]], eigenvalues: list[tuple[torch.Tensor]], deltas: torch.Tensor | None = None, damping: bool = False) -
-
-

Decomposed Kronecker factored approximate curvature representation -for a corresponding neural network. -Each matrix in Kron is decomposed to obtain KronDecomposed. -Front-loading decomposition allows cheap repeated computation -of inverses and log determinants. -In contrast to Kron, we can add scalar or layerwise scalars but -we cannot add other Kron or KronDecomposed anymore.

-

Parameters

-
-
eigenvectors : list[Tuple[torch.Tensor]]
-
eigenvectors corresponding to matrices in a corresponding Kron
-
eigenvalues : list[Tuple[torch.Tensor]]
-
eigenvalues corresponding to matrices in a corresponding Kron
-
deltas : torch.Tensor
-
addend for each group of Kronecker factors representing, for example, -a prior precision
-
dampen : bool, default=False
-
use dampen approximation mixing prior and Kron partially multiplicatively
-
-

Methods

-
-
-def detach(self) ‑> KronDecomposed -
-
-
-
-
-def logdet(self) ‑> torch.Tensor -
-
-

Compute log determinant of the Kronecker factors and sums them up. -This corresponds to the log determinant of the entire Hessian approximation. -In contrast to Kron.logdet(), additive deltas corresponding to prior -precisions are added.

-

Returns

-
-
logdet : torch.Tensor
-
 
-
-
-
-def inv_square_form(self, W: torch.Tensor) ‑> torch.Tensor -
-
-
-
-
-def bmm(self, W: torch.Tensor, exponent: float = -1) ‑> torch.Tensor -
-
-

Batched matrix multiplication with the decomposed Kronecker factors. -This is useful for computing the predictive or a regularization loss. -Compared to Kron.bmm(), a prior can be added here in form of deltas -and the exponent can be other than just 1. -Computes H^{exponent} W.

-

Parameters

-
-
W : torch.Tensor
-
matrix (batch, classes, params)
-
exponent : float, default=1
-
 
-
-

Returns

-
-
SW : torch.Tensor
-
result (batch, classes, params)
-
-
-
-def diag(self, exponent: float = 1) ‑> torch.Tensor -
-
-

Extract diagonal of the entire decomposed Kronecker factorization.

-

Parameters

-
-
exponent : float, default=1
-
exponent of the Kronecker factorization
-
-

Returns

-
-
diag : torch.Tensor
-
 
-
-
-
-def to_matrix(self, exponent: float = 1) ‑> torch.Tensor -
-
-

Make the Kronecker factorization dense by computing the kronecker product. -Warning: this should only be used for testing purposes as it will allocate -large amounts of memory for big architectures.

-

Parameters

-
-
exponent : float, default=1
-
exponent of the Kronecker factorization
-
-

Returns

-
-
block_diag : torch.Tensor
-
 
-
-
-
-
-
-
-
- -
- - - diff --git a/docs/utils/metrics.html b/docs/utils/metrics.html deleted file mode 100644 index ff60fcb8..00000000 --- a/docs/utils/metrics.html +++ /dev/null @@ -1,139 +0,0 @@ - - - - - - -laplace.utils.metrics API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.utils.metrics

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class RunningNLLMetric -(ignore_index: int = -100) -
-
-

NLL metrics that

-

Parameters

-
-
ignore_index : int, default = -100
-
which class label to ignore when computing the NLL loss
-
-

Initialize internal Module state, shared by both nn.Module and ScriptModule.

-

Ancestors

-
    -
  • torchmetrics.metric.Metric
    • -
  • torch.nn.modules.module.Module
    • -
  • abc.ABC
    • -
-

Class variables

-
-
var is_differentiable : Optional[bool]
-
-
-
-
var higher_is_better : Optional[bool]
-
-
-
-
var full_state_update : Optional[bool]
-
-
-
-
var plot_lower_bound : Optional[float]
-
-
-
-
var plot_upper_bound : Optional[float]
-
-
-
-
var plot_legend_name : Optional[str]
-
-
-
-
-

Methods

-
-
-def update(self, probs: torch.Tensor, targets: torch.Tensor) ‑> None -
-
-

Parameters

-
-
probs : torch.Tensor
-
probability tensor of shape (…, n_classes)
-
targets : torch.Tensor
-
integer tensor of shape (…)
-
-
-
-def compute(self) ‑> torch.Tensor -
-
-

Override this method to compute the final metric value.

-

This method will automatically synchronize state variables when running in distributed backend.

-
-
-
-
-
-
- -
- - - diff --git a/docs/utils/subnetmask.html b/docs/utils/subnetmask.html deleted file mode 100644 index 06632728..00000000 --- a/docs/utils/subnetmask.html +++ /dev/null @@ -1,469 +0,0 @@ - - - - - - -laplace.utils.subnetmask API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.utils.subnetmask

-
-
-
-
-
-
-
-
-
-
-

Classes

-
-
-class SubnetMask -(model: nn.Module) -
-
-

Baseclass for all subnetwork masks in this library (for subnetwork Laplace).

-

Parameters

-
-
model : torch.nn.Module
-
 
-
-

Subclasses

- -

Instance variables

-
-
prop indices : torch.LongTensor
-
-
-
-
prop n_params_subnet : int
-
-
-
-
-

Methods

-
-
-def convert_subnet_mask_to_indices(self, subnet_mask: torch.Tensor) ‑> torch.LongTensor -
-
-

Converts a subnetwork mask into subnetwork indices.

-

Parameters

-
-
subnet_mask : torch.Tensor
-
a binary vector of size (n_params) where 1s locate the subnetwork parameters -within the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters()))
-
-

Returns

-
-
subnet_mask_indices : torch.LongTensor
-
a vector of indices of the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters())) -that define the subnetwork
-
-
-
-def select(self, train_loader: DataLoader | None = None) -
-
-

Select the subnetwork mask.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader, default=None
-
each iterate is a training batch (X, y); -train_loader.dataset needs to be set to access N, size of the data set
-
-

Returns

-
-
subnet_mask_indices : torch.LongTensor
-
a vector of indices of the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters())) -that define the subnetwork
-
-
-
-def get_subnet_mask(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-

Get the subnetwork mask.

-

Parameters

-
-
train_loader : torch.data.utils.DataLoader
-
each iterate is a training batch (X, y); -train_loader.dataset needs to be set to access N, size of the data set
-
-

Returns

-
-
subnet_mask : torch.Tensor
-
a binary vector of size (n_params) where 1s locate the subnetwork parameters -within the vectorized model parameters -(i.e. torch.nn.utils.parameters_to_vector(model.parameters()))
-
-
-
-
-
-class RandomSubnetMask -(model: nn.Module, n_params_subnet: int) -
-
-

Subnetwork mask of parameters sampled uniformly at random.

-

Ancestors

-
    -
  • laplace.utils.subnetmask.ScoreBasedSubnetMask
    • -
  • SubnetMask
    • -
-

Methods

-
-
-def compute_param_scores(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class LargestMagnitudeSubnetMask -(model: nn.Module, n_params_subnet: int) -
-
-

Subnetwork mask identifying the parameters with the largest magnitude.

-

Ancestors

-
    -
  • laplace.utils.subnetmask.ScoreBasedSubnetMask
    • -
  • SubnetMask
    • -
-

Methods

-
-
-def compute_param_scores(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class LargestVarianceDiagLaplaceSubnetMask -(model: nn.Module, n_params_subnet: int, diag_laplace_model: DiagLaplace) -
-
-

Subnetwork mask identifying the parameters with the largest marginal variances -(estimated using a diagonal Laplace approximation over all model parameters).

-

Parameters

-
-
model : torch.nn.Module
-
 
-
n_params_subnet : int
-
number of parameters in the subnetwork (i.e. number of top-scoring parameters to select)
-
diag_laplace_model : DiagLaplace
-
diagonal Laplace model to use for variance estimation
-
-

Ancestors

-
    -
  • laplace.utils.subnetmask.ScoreBasedSubnetMask
    • -
  • SubnetMask
    • -
-

Methods

-
-
-def compute_param_scores(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class LargestVarianceSWAGSubnetMask -(model: nn.Module, n_params_subnet: int, likelihood: Likelihood | str = Likelihood.CLASSIFICATION, swag_n_snapshots: int = 40, swag_snapshot_freq: int = 1, swag_lr: float = 0.01) -
-
-

Subnetwork mask identifying the parameters with the largest marginal variances -(estimated using diagonal SWAG over all model parameters).

-

Parameters

-
-
model : torch.nn.Module
-
 
-
n_params_subnet : int
-
number of parameters in the subnetwork (i.e. number of top-scoring parameters to select)
-
likelihood : str
-
'classification' or 'regression'
-
swag_n_snapshots : int
-
number of model snapshots to collect for SWAG
-
swag_snapshot_freq : int
-
SWAG snapshot collection frequency (in epochs)
-
swag_lr : float
-
learning rate for SWAG snapshot collection
-
-

Ancestors

-
    -
  • laplace.utils.subnetmask.ScoreBasedSubnetMask
    • -
  • SubnetMask
    • -
-

Methods

-
-
-def compute_param_scores(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-
-
-
-

Inherited members

- -
-
-class ParamNameSubnetMask -(model: nn.Module, parameter_names: list[str]) -
-
-

Subnetwork mask corresponding to the specified parameters of the neural network.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
parameter_names : List[str]
-
list of names of the parameters (as in model.named_parameters()) -that define the subnetwork
-
-

Ancestors

- -

Methods

-
-
-def get_subnet_mask(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-

Get the subnetwork mask identifying the specified parameters.

-
-
-

Inherited members

- -
-
-class ModuleNameSubnetMask -(model: nn.Module, module_names: list[str]) -
-
-

Subnetwork mask corresponding to the specified modules of the neural network.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
parameter_names : List[str]
-
list of names of the modules (as in model.named_modules()) that define the subnetwork; -the modules cannot have children, i.e. need to be leaf modules
-
-

Ancestors

- -

Subclasses

- -

Methods

-
-
-def get_subnet_mask(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-

Get the subnetwork mask identifying the specified modules.

-
-
-

Inherited members

- -
-
-class LastLayerSubnetMask -(model: nn.Module, last_layer_name: str | None = None) -
-
-

Subnetwork mask corresponding to the last layer of the neural network.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
last_layer_name : str, default=None
-
name of the model's last layer, if None it will be determined automatically
-
-

Ancestors

- -

Methods

-
-
-def get_subnet_mask(self, train_loader: DataLoader) ‑> torch.Tensor -
-
-

Get the subnetwork mask identifying the last layer.

-
-
-

Inherited members

- -
-
-
-
- -
- - - diff --git a/docs/utils/swag.html b/docs/utils/swag.html deleted file mode 100644 index 69d02ed2..00000000 --- a/docs/utils/swag.html +++ /dev/null @@ -1,105 +0,0 @@ - - - - - - -laplace.utils.swag API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.utils.swag

-
-
-
-
-
-
-
-
-

Functions

-
-
-def fit_diagonal_swag_var(model: nn.Module, train_loader: DataLoader, criterion: nn.CrossEntropyLoss | nn.MSELoss, n_snapshots_total: int = 40, snapshot_freq: int = 1, lr: float = 0.01, momentum: float = 0.9, weight_decay: float = 0.0003, min_var: float = 1e-30) -
-
-

Fit diagonal SWAG [1], which estimates marginal variances of model parameters by -computing the first and second moment of SGD iterates with a large learning rate.

-

Implementation partly adapted from: -- https://github.com/wjmaddox/swa_gaussian/blob/master/swag/posteriors/swag.py -- https://github.com/wjmaddox/swa_gaussian/blob/master/experiments/train/run_swag.py

-

References

-

[1] Maddox, W., Garipov, T., Izmailov, P., Vetrov, D., Wilson, AG. -A Simple Baseline for Bayesian Uncertainty in Deep Learning. -NeurIPS 2019.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
train_loader : torch.data.utils.DataLoader
-
training data loader to use for snapshot collection
-
criterion : torch.nn.CrossEntropyLoss or torch.nn.MSELoss
-
loss function to use for snapshot collection
-
n_snapshots_total : int
-
total number of model snapshots to collect
-
snapshot_freq : int
-
snapshot collection frequency (in epochs)
-
lr : float
-
SGD learning rate for collecting snapshots
-
momentum : float
-
SGD momentum
-
weight_decay : float
-
SGD weight decay
-
min_var : float
-
minimum parameter variance to clamp to (for numerical stability)
-
-

Returns

-
-
param_variances : torch.Tensor
-
vector of marginal variances for each model parameter
-
-
-
-
-
-
-
- -
- - - diff --git a/docs/utils/utils.html b/docs/utils/utils.html deleted file mode 100644 index 8e085d13..00000000 --- a/docs/utils/utils.html +++ /dev/null @@ -1,268 +0,0 @@ - - - - - - -laplace.utils.utils API documentation - - - - - - - - - - - - - -
-
-
-

Module laplace.utils.utils

-
-
-
-
-
-
-
-
-

Functions

-
-
-def get_nll(out_dist: torch.Tensor, targets: torch.Tensor) ‑> torch.Tensor -
-
-
-
-
-def validate(laplace: BaseLaplace, val_loader: DataLoader, loss: torchmetrics.Metric | Callable[[torch.Tensor, torch.Tensor], torch.Tensor] | Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor], pred_type: PredType | str = PredType.GLM, link_approx: LinkApprox | str = LinkApprox.PROBIT, n_samples: int = 100, dict_key_y: str = 'labels') -
-
-
-
-
-def parameters_per_layer(model: nn.Module) ‑> list[int] -
-
-

Get number of parameters per layer.

-

Parameters

-
-
model : torch.nn.Module
-
 
-
-

Returns

-
-
params_per_layer : list[int]
-
 
-
-
-
-def invsqrt_precision(M: torch.Tensor) ‑> torch.Tensor -
-
-

Compute M^{-0.5} as a tridiagonal matrix.

-

Parameters

-
-
M : torch.Tensor
-
 
-
-

Returns

-
-
M_invsqrt : torch.Tensor
-
 
-
-
-
-def kron(t1: torch.Tensor, t2: torch.Tensor) ‑> torch.Tensor -
-
-

Computes the Kronecker product between two tensors.

-

Parameters

-
-
t1 : torch.Tensor
-
 
-
t2 : torch.Tensor
-
 
-
-

Returns

-
-
kron_product : torch.Tensor
-
 
-
-
-
-def diagonal_add_scalar(X: torch.Tensor, value: torch.Tensor) ‑> torch.Tensor -
-
-

Add scalar value value to diagonal of X.

-

Parameters

-
-
X : torch.Tensor
-
 
-
value : torch.Tensor or float
-
 
-
-

Returns

-
-
X_add_scalar : torch.Tensor
-
 
-
-
-
-def symeig(M: torch.Tensor) ‑> tuple[torch.Tensor, torch.Tensor] -
-
-

Symetric eigendecomposition avoiding failure cases by -adding and removing jitter to the diagonal.

-

Parameters

-
-
M : torch.Tensor
-
 
-
-

Returns

-
-
L : torch.Tensor
-
eigenvalues
-
W : torch.Tensor
-
eigenvectors
-
-
-
-def block_diag(blocks: list[torch.Tensor]) ‑> torch.Tensor -
-
-

Compose block-diagonal matrix of individual blocks.

-

Parameters

-
-
blocks : list[torch.Tensor]
-
 
-
-

Returns

-
-
M : torch.Tensor
-
 
-
-
-
-def expand_prior_precision(prior_prec: torch.Tensor, model: nn.Module) ‑> torch.Tensor -
-
-

Expand prior precision to match the shape of the model parameters.

-

Parameters

-
-
prior_prec : torch.Tensor 1-dimensional
-
prior precision
-
model : torch.nn.Module
-
torch model with parameters that are regularized by prior_prec
-
-

Returns

-
-
expanded_prior_prec : torch.Tensor
-
expanded prior precision has the same shape as model parameters
-
-
-
-
-
-

Classes

-
-
-class SoDSampler -(N, M, seed: int = 0) -
-
-

Base class for all Samplers.

-

Every Sampler subclass has to provide an :meth:__iter__ method, providing a -way to iterate over indices or lists of indices (batches) of dataset elements, -and may provide a :meth:__len__ method that returns the length of the returned iterators.

-

Args

-
-
data_source : Dataset
-
This argument is not used and will be removed in 2.2.0. -You may still have custom implementation that utilizes it.
-
-

Example

-
>>> # xdoctest: +SKIP
->>> class AccedingSequenceLengthSampler(Sampler[int]):
->>>     def __init__(self, data: List[str]) -> None:
->>>         self.data = data
->>>
->>>     def __len__(self) -> int:
->>>         return len(self.data)
->>>
->>>     def __iter__(self) -> Iterator[int]:
->>>         sizes = torch.tensor([len(x) for x in self.data])
->>>         yield from torch.argsort(sizes).tolist()
->>>
->>> class AccedingSequenceLengthBatchSampler(Sampler[List[int]]):
->>>     def __init__(self, data: List[str], batch_size: int) -> None:
->>>         self.data = data
->>>         self.batch_size = batch_size
->>>
->>>     def __len__(self) -> int:
->>>         return (len(self.data) + self.batch_size - 1) // self.batch_size
->>>
->>>     def __iter__(self) -> Iterator[List[int]]:
->>>         sizes = torch.tensor([len(x) for x in self.data])
->>>         for batch in torch.chunk(torch.argsort(sizes), len(self)):
->>>             yield batch.tolist()
-
-
-

Note: The :meth:__len__ method isn't strictly required by

-

:class:~torch.utils.data.DataLoader, but is expected in any -calculation involving the length of a :class:~torch.utils.data.DataLoader.

-
-

Ancestors

-
    -
  • torch.utils.data.sampler.Sampler
    • -
  • typing.Generic
    • -
-
-
-
-
- -
- - - diff --git a/examples/helper/util.py b/examples/helper/util.py index 6f1005dc..cfb6da07 100644 --- a/examples/helper/util.py +++ b/examples/helper/util.py @@ -47,4 +47,4 @@ def plot_regression( if plot: plt.show() else: - plt.savefig(f"docs/{file_name}.png") + plt.savefig(f"docs/assets/{file_name}.png") diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..bf5db09a --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,88 @@ +site_name: Laplace Approximations in PyTorch +theme: + name: material + # logo: assets/laplace_logo.png + features: + - navigation.instant + - navigation.instant.prefetch + - navigation.instant.progress + - navigation.sections + - navigation.indexes + - navigation.tracking + - content.code.annotate + - toc.follow + - navigation.footer + - navigation.top + - content.code.copy + - content.tabs.link + palette: + # Palette toggle for automatic mode + - media: "(prefers-color-scheme)" + primary: black + toggle: + icon: material/brightness-auto + name: Switch to light mode + + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + primary: black + scheme: default + toggle: + icon: material/brightness-7 + name: Switch to dark mode + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + primary: black + scheme: slate + toggle: + icon: material/brightness-4 + name: Switch to system preference +repo_url: https://github.com/aleximmer/laplace +repo_name: laplace +site_author: wiseodd +site_url: https://aleximmer.github.io/Laplace/ +site_dir: site/laplace +markdown_extensions: + - admonition + - pymdownx.details + - toc: + permalink: "#" + - pymdownx.snippets: + - pymdownx.magiclink: + - attr_list: + - md_in_html: + - pymdownx.inlinehilite: + - pymdownx.superfences: + - markdown.extensions.attr_list: + - pymdownx.keys: + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.tabbed: + alternate_style: true + - pymdownx.arithmatex: + generic: true +plugins: + - search +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/aleximmer/laplace + - icon: fontawesome/brands/python + link: https://pypi.org/project/laplace-torch/ +extra_javascript: + - javascripts/mathjax.js + - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js +nav: + - Introduction: index.md + - Regression Example: regression_example.md + - Calibration Example: calibration_example.md + - GP Inference Example: calibration_gp_example.md + - Huggingface LLMs Example: huggingface_example.md + - Reward Modeling Example: reward_modeling_example.md +validation: + omitted_files: warn + absolute_links: warn + unrecognized_links: warn diff --git a/pyproject.toml b/pyproject.toml index 6c2865f4..2e3cb0a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,9 @@ py-modules = ["laplace"] dev = [ "coveralls", "matplotlib", - "pdoc3", + "mkdocs", + "mkdocs-material", + "mkdocstrings-python", "pytest", "pytest-cov", "pytest-mock", From baf6065d10e0fe78a68ce1444f1250abf84d6379 Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Fri, 13 Sep 2024 14:48:29 -0400 Subject: [PATCH 02/10] Add images to the regression example docs --- docs/regression_example.md | 6 ++---- mkdocs.yml | 16 ++++++++-------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/regression_example.md b/docs/regression_example.md index 778d7c0f..ef43cb76 100644 --- a/docs/regression_example.md +++ b/docs/regression_example.md @@ -88,8 +88,7 @@ pred_std = np.sqrt(f_sigma**2 + la.sigma_noise.item()**2) plot_regression(X_train, y_train, x, f_mu, pred_std) ``` -.. image:: regression_example.png -:align: center +![Posthoc Laplace](assets/regression_example.png) In comparison to the MAP, the predictive shows useful uncertainties. When our MAP is over or underfit, the Laplace approximation cannot fix this anymore. @@ -118,5 +117,4 @@ pred_std = np.sqrt(f_sigma**2 + la.sigma_noise.item()**2) plot_regression(X_train, y_train, x, f_mu, pred_std) ``` -.. image:: regression_example_online.png -:align: center +![Online Laplace](assets/regression_example_online.png) diff --git a/mkdocs.yml b/mkdocs.yml index bf5db09a..6efb15c8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -48,14 +48,13 @@ markdown_extensions: - pymdownx.details - toc: permalink: "#" - - pymdownx.snippets: - - pymdownx.magiclink: - - attr_list: - - md_in_html: - - pymdownx.inlinehilite: - - pymdownx.superfences: - - markdown.extensions.attr_list: - - pymdownx.keys: + - pymdownx.snippets + - pymdownx.magiclink + - attr_list + - md_in_html + - pymdownx.inlinehilite + - pymdownx.superfences + - pymdownx.keys - pymdownx.tasklist: custom_checkbox: true - pymdownx.highlight: @@ -66,6 +65,7 @@ markdown_extensions: generic: true plugins: - search + - mkdocstrings extra: social: - icon: fontawesome/brands/github From 9e87cb49c6c3100d37112fe0ddf125e85ffdbc64 Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Fri, 13 Sep 2024 16:03:58 -0400 Subject: [PATCH 03/10] Add API reference to docs --- docs/api_reference/baselaplace.md | 4 ++++ docs/api_reference/curvatures.md | 17 +++++++++++++++++ docs/api_reference/functionallaplace.md | 4 ++++ docs/api_reference/laplace.md | 4 ++++ docs/api_reference/lllaplace.md | 7 +++++++ docs/api_reference/marglik_training.md | 1 + docs/api_reference/parametriclaplace.md | 8 ++++++++ docs/api_reference/subnetlaplace.md | 6 ++++++ docs/api_reference/utils.md | 1 + laplace/baselaplace.py | 25 +++++++++++++++++-------- laplace/lllaplace.py | 6 ++++-- laplace/subnetlaplace.py | 6 ++++-- mkdocs.yml | 19 ++++++++++++++++++- 13 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 docs/api_reference/baselaplace.md create mode 100644 docs/api_reference/curvatures.md create mode 100644 docs/api_reference/functionallaplace.md create mode 100644 docs/api_reference/laplace.md create mode 100644 docs/api_reference/lllaplace.md create mode 100644 docs/api_reference/marglik_training.md create mode 100644 docs/api_reference/parametriclaplace.md create mode 100644 docs/api_reference/subnetlaplace.md create mode 100644 docs/api_reference/utils.md diff --git a/docs/api_reference/baselaplace.md b/docs/api_reference/baselaplace.md new file mode 100644 index 00000000..247af2c0 --- /dev/null +++ b/docs/api_reference/baselaplace.md @@ -0,0 +1,4 @@ +::: laplace.baselaplace + options: + members: + - BaseLaplace diff --git a/docs/api_reference/curvatures.md b/docs/api_reference/curvatures.md new file mode 100644 index 00000000..ea62ce18 --- /dev/null +++ b/docs/api_reference/curvatures.md @@ -0,0 +1,17 @@ +::: laplace.curvature + options: + members: + - CurvatureInterface + - GGNInterface + - EFInterface + - AsdlInterface + - AsdlGGN + - AsdlEF + - AsdlHessian + - BackPackInterface + - BackPackGGN + - BackPackEF + - CurvlinopsInterface + - CurvlinopsGGN + - CurvlinopsEF + - CurvlinopsHessian diff --git a/docs/api_reference/functionallaplace.md b/docs/api_reference/functionallaplace.md new file mode 100644 index 00000000..b687aa73 --- /dev/null +++ b/docs/api_reference/functionallaplace.md @@ -0,0 +1,4 @@ +::: laplace.baselaplace + options: + members: + - FunctionalLaplace diff --git a/docs/api_reference/laplace.md b/docs/api_reference/laplace.md new file mode 100644 index 00000000..2d3f1031 --- /dev/null +++ b/docs/api_reference/laplace.md @@ -0,0 +1,4 @@ +::: laplace.laplace + options: + members: + - Laplace diff --git a/docs/api_reference/lllaplace.md b/docs/api_reference/lllaplace.md new file mode 100644 index 00000000..6dcee884 --- /dev/null +++ b/docs/api_reference/lllaplace.md @@ -0,0 +1,7 @@ +::: laplace.lllaplace + options: + members: + - LLLaplace + - DiagLLLaplace + - KronLLLaplace + - FullLLLaplace diff --git a/docs/api_reference/marglik_training.md b/docs/api_reference/marglik_training.md new file mode 100644 index 00000000..5e2e7f4d --- /dev/null +++ b/docs/api_reference/marglik_training.md @@ -0,0 +1 @@ +::: laplace.marglik_training diff --git a/docs/api_reference/parametriclaplace.md b/docs/api_reference/parametriclaplace.md new file mode 100644 index 00000000..eb8c12f2 --- /dev/null +++ b/docs/api_reference/parametriclaplace.md @@ -0,0 +1,8 @@ +::: laplace.baselaplace + options: + members: + - ParametricLaplace + - DiagLaplace + - KronLaplace + - LowRankLaplace + - FullLaplace diff --git a/docs/api_reference/subnetlaplace.md b/docs/api_reference/subnetlaplace.md new file mode 100644 index 00000000..9c65a3aa --- /dev/null +++ b/docs/api_reference/subnetlaplace.md @@ -0,0 +1,6 @@ +::: laplace.subnetlaplace + options: + members: + - SubnetLaplace + - DiagSubnetLaplace + - FullSubnetLaplace diff --git a/docs/api_reference/utils.md b/docs/api_reference/utils.md new file mode 100644 index 00000000..10724bf0 --- /dev/null +++ b/docs/api_reference/utils.md @@ -0,0 +1 @@ +::: laplace.utils diff --git a/laplace/baselaplace.py b/laplace/baselaplace.py index c8909b1f..f8296e9c 100644 --- a/laplace/baselaplace.py +++ b/laplace/baselaplace.py @@ -723,10 +723,12 @@ class ParametricLaplace(BaseLaplace): a Gaussian distribution \\(\\mathcal{N}(\\theta_{MAP}, P^{-1})\\). The goal of this class is to compute the posterior precision \\(P\\) which sums as - \\[ + + $$ P = \\sum_{n=1}^N \\nabla^2_\\theta \\log p(\\mathcal{D}_n \\mid \\theta) \\vert_{\\theta_{MAP}} + \\nabla^2_\\theta \\log p(\\theta) \\vert_{\\theta_{MAP}}. - \\] + $$ + Every subclass implements different approximations to the log likelihood Hessians, for example, a diagonal one. The prior is assumed to be Gaussian and therefore we have a simple form for \\(\\nabla^2_\\theta \\log p(\\theta) \\vert_{\\theta_{MAP}} = P_0 \\). @@ -892,9 +894,10 @@ def log_det_posterior_precision(self) -> torch.Tensor: @property def log_det_ratio(self) -> torch.Tensor: """Compute the log determinant ratio, a part of the log marginal likelihood. - \\[ + + $$ \\log \\frac{\\det P}{\\det P_0} = \\log \\det P - \\log \\det P_0 - \\] + $$ Returns ------- @@ -904,9 +907,11 @@ def log_det_ratio(self) -> torch.Tensor: def square_norm(self, value) -> torch.Tensor: """Compute the square norm under post. Precision with `value-self.mean` as 𝛥: - \\[ + + $$ \\Delta^\top P \\Delta - \\] + $$ + Returns ------- square_form @@ -2420,7 +2425,9 @@ def gp_kernel_prior_variance(self): def functional_variance(self, Js_star: torch.Tensor) -> torch.Tensor: """GP posterior variance: - \\[ k_{**} - K_{*M} (K_{MM}+ L_{MM}^{-1})^{-1} K_{M*}\\] + $$ + k_{**} - K_{*M} (K_{MM}+ L_{MM}^{-1})^{-1} K_{M*} + $$ Parameters ---------- @@ -2457,7 +2464,9 @@ def functional_variance(self, Js_star: torch.Tensor) -> torch.Tensor: def functional_covariance(self, Js_star: torch.Tensor) -> torch.Tensor: """GP posterior covariance: - \\[ k_{**} - K_{*M} (K_{MM}+ L_{MM}^{-1})^{-1} K_{M*}\\] + $$ + k_{**} - K_{*M} (K_{MM}+ L_{MM}^{-1})^{-1} K_{M*} + $$ Parameters ---------- diff --git a/laplace/lllaplace.py b/laplace/lllaplace.py index bea33e68..7529870c 100644 --- a/laplace/lllaplace.py +++ b/laplace/lllaplace.py @@ -43,10 +43,12 @@ class LLLaplace(ParametricLaplace): are treated probabilistically. The goal of this class is to compute the posterior precision \\(P\\) which sums as - \\[ + + $$ P = \\sum_{n=1}^N \\nabla^2_\\theta \\log p(\\mathcal{D}_n \\mid \\theta) \\vert_{\\theta_{MAP}} + \\nabla^2_\\theta \\log p(\\theta) \\vert_{\\theta_{MAP}}. - \\] + $$ + Every subclass implements different approximations to the log likelihood Hessians, for example, a diagonal one. The prior is assumed to be Gaussian and therefore we have a simple form for \\(\\nabla^2_\\theta \\log p(\\theta) \\vert_{\\theta_{MAP}} = P_0 \\). diff --git a/laplace/subnetlaplace.py b/laplace/subnetlaplace.py index 01685b9f..0ec3c28a 100644 --- a/laplace/subnetlaplace.py +++ b/laplace/subnetlaplace.py @@ -24,10 +24,12 @@ class SubnetLaplace(ParametricLaplace): neural network) are treated probabilistically. The goal of this class is to compute the posterior precision \\(P\\) which sums as - \\[ + + $$ P = \\sum_{n=1}^N \\nabla^2_\\theta \\log p(\\mathcal{D}_n \\mid \\theta) \\vert_{\\theta_{MAP}} + \\nabla^2_\\theta \\log p(\\theta) \\vert_{\\theta_{MAP}}. - \\] + $$ + The prior is assumed to be Gaussian and therefore we have a simple form for \\(\\nabla^2_\\theta \\log p(\\theta) \\vert_{\\theta_{MAP}} = P_0 \\). In particular, we assume a scalar or diagonal prior precision so that in diff --git a/mkdocs.yml b/mkdocs.yml index 6efb15c8..96fe3201 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -65,7 +65,14 @@ markdown_extensions: generic: true plugins: - search - - mkdocstrings + - mkdocstrings: + enabled: true + default_handler: python + handlers: + python: + options: + docstring_style: numpy + show_root_heading: false extra: social: - icon: fontawesome/brands/github @@ -82,6 +89,16 @@ nav: - GP Inference Example: calibration_gp_example.md - Huggingface LLMs Example: huggingface_example.md - Reward Modeling Example: reward_modeling_example.md + - API Reference: + - Laplace Frontend: api_reference/laplace.md + - Base Laplace: api_reference/baselaplace.md + - Parametric Laplace: api_reference/parametriclaplace.md + - Functional Laplace: api_reference/functionallaplace.md + - Last-Layer Laplace: api_reference/lllaplace.md + - Subnet Laplace: api_reference/subnetlaplace.md + - Curvatures: api_reference/curvatures.md + - Marglik Training Utils: api_reference/marglik_training.md + - Utilities: api_reference/utils.md validation: omitted_files: warn absolute_links: warn From 682f67df3ab40ef0354e54b7caa50ca9894caede Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Fri, 13 Sep 2024 16:10:16 -0400 Subject: [PATCH 04/10] Update README --- README.md | 19 ++- uv.lock | 432 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 421 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index bc0a4d16..fac194fd 100644 --- a/README.md +++ b/README.md @@ -450,17 +450,16 @@ For example, currently the [`curvature.CurvlinopsInterface`](https://github.com/ ## Documentation -The documentation is available [here](https://aleximmer.github.io/Laplace) or can be generated and/or viewed locally: +The documentation is available [here](https://aleximmer.github.io/Laplace) or can be +generated and/or viewed locally: **With `uv`** ```bash # assuming the repository was cloned uv sync --all-extras -# create docs and write to html -uv run bash update_docs.sh -# .. or serve the docs directly -uv run pdoc --http 0.0.0.0:8080 laplace --template-dir template +# serve the docs on a liveserver +uv run mkdocs serve ``` **With `pip`** @@ -468,10 +467,8 @@ uv run pdoc --http 0.0.0.0:8080 laplace --template-dir template ```bash # assuming the repository was cloned pip install -e ".[dev]" -# create docs and write to html -bash update_docs.sh -# .. or serve the docs directly -pdoc --http 0.0.0.0:8080 laplace --template-dir template +# serve the docs on a liveserver +mkdocs serve ``` ## Contributing @@ -481,7 +478,9 @@ Pull requests are very welcome. Please follow these guidelines: 1. Follow the [development setup](#setup-dev-environment). 2. Use [ruff](https://github.com/astral-sh/ruff) as autoformatter. Please refer to the following [makefile](https://github.com/aleximmer/Laplace/blob/main/makefile) and run it via `make ruff`. Please note that the order of `ruff check --fix` and `ruff format` is important! 3. Also use [ruff](https://github.com/astral-sh/ruff) as linter. Please manually fix all linting errors/warnings before opening a pull request. -4. Fully document your changes in the form of Python docstrings, typehinting, and (if applicable) code/markdown examples in the `./examples` subdirectory. +4. Fully document your changes in the form of Python docstrings, typehinting. +5. Add tutorial-style documentation in both `docs/` (in the form of markdown files) + and `examples/` (in the form of runnable scripts) dirs. 5. Provide as many test cases as possible. Make sure all test cases pass. Issues, bug reports, and ideas are also very welcome! diff --git a/uv.lock b/uv.lock index 9b6b78fe..2b433c3b 100644 --- a/uv.lock +++ b/uv.lock @@ -14,6 +14,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/9a/7548968acfacb162c88dd9c2247a54f36136221ebb13d457016b28d47523/asdfghjkl-0.1a4-py3-none-any.whl", hash = "sha256:9cedec38494d7907075941d03d018d8c2b64047e6c41713a49f30e3790816c51", size = 89694 }, ] +[[package]] +name = "babel" +version = "2.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2a/74/f1bc80f23eeba13393b7222b11d95ca3af2c1e28edca18af487137eefed9/babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316", size = 9348104 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/20/bc79bc575ba2e2a7f70e8a1155618bb1301eaa5132a8271373a6903f73f8/babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b", size = 9587599 }, +] + [[package]] name = "backpack-for-pytorch" version = "1.6.0" @@ -107,6 +116,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/76/e6222113b83e3622caa4bb41032d0b1bf785250607392e1b778aca0b8a7d/charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", size = 48543 }, ] +[[package]] +name = "click" +version = "8.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "platform_system == 'Windows'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", size = 97941 }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -380,6 +401,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/44/73bea497ac69bafde2ee4269292fa3b41f1198f4bb7bbaaabde30ad29d4a/fsspec-2024.6.1-py3-none-any.whl", hash = "sha256:3cb443f8bcd2efb31295a5b9fdb02aee81d8452c80d28f97a6d0959e6cee101e", size = 177561 }, ] +[[package]] +name = "ghp-import" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034 }, +] + +[[package]] +name = "griffe" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/d1/dcd486d6d577cb12490c78cfa88679fb9b481b227807f14632ba9bd82245/griffe-1.3.1.tar.gz", hash = "sha256:3f86a716b631a4c0f96a43cb75d05d3c85975003c20540426c0eba3b0581c56a", size = 382412 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/19/63971981a20aecfa7cbd07c5cac6914cf1180b3dd8db5fe8ab2ea410315f/griffe-1.3.1-py3-none-any.whl", hash = "sha256:940aeb630bc3054b4369567f150b6365be6f11eef46b0ed8623aea96e6d17b19", size = 126902 }, +] + [[package]] name = "idna" version = "3.7" @@ -539,7 +584,9 @@ dependencies = [ dev = [ { name = "coveralls" }, { name = "matplotlib" }, - { name = "pdoc3" }, + { name = "mkdocs" }, + { name = "mkdocs-material" }, + { name = "mkdocstrings-python" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "pytest-mock" }, @@ -554,9 +601,11 @@ requires-dist = [ { name = "coveralls", marker = "extra == 'dev'" }, { name = "curvlinops-for-pytorch", specifier = ">=2.0" }, { name = "matplotlib", marker = "extra == 'dev'" }, + { name = "mkdocs", marker = "extra == 'dev'" }, + { name = "mkdocs-material", marker = "extra == 'dev'" }, + { name = "mkdocstrings-python", marker = "extra == 'dev'" }, { name = "numpy" }, { name = "opt-einsum" }, - { name = "pdoc3", marker = "extra == 'dev'" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pytest-cov", marker = "extra == 'dev'" }, { name = "pytest-mock", marker = "extra == 'dev'" }, @@ -582,18 +631,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/d5/ed204bc738672c17455019b5e0c7c8d1effb0ea17707150ca50336298ca0/lightning_utilities-0.11.6-py3-none-any.whl", hash = "sha256:ecd9953c316cbaf56ad820fbe7bd062187b9973c4a23d47b076cd59dc080a310", size = 26818 }, ] -[[package]] -name = "mako" -version = "1.3.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/67/03/fb5ba97ff65ce64f6d35b582aacffc26b693a98053fa831ab43a437cbddb/Mako-1.3.5.tar.gz", hash = "sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc", size = 392738 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/62/70f5a0c2dd208f9f3f2f9afd103aec42ee4d9ad2401d78342f75e9b8da36/Mako-1.3.5-py3-none-any.whl", hash = "sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a", size = 78565 }, -] - [[package]] name = "markdown" version = "3.7" @@ -713,6 +750,135 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/e8/a69f4ad5b544f509ec3718dfa003187a94a37d79bf2e175180668c0ff8ec/matplotlib-3.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:050598c2b29e0b9832cde72bcf97627bf00262adbc4a54e2b856426bb2ef0697", size = 7845515 }, ] +[[package]] +name = "mergedeep" +version = "1.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354 }, +] + +[[package]] +name = "mkdocs" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, + { name = "ghp-import" }, + { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mergedeep" }, + { name = "mkdocs-get-deps" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "pyyaml" }, + { name = "pyyaml-env-tag" }, + { name = "watchdog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451 }, +] + +[[package]] +name = "mkdocs-autorefs" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mkdocs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/ae/0f1154c614d6a8b8a36fff084e5b82af3a15f7d2060cf0dcdb1c53297a71/mkdocs_autorefs-1.2.0.tar.gz", hash = "sha256:a86b93abff653521bda71cf3fc5596342b7a23982093915cb74273f67522190f", size = 40262 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/26/4d39d52ea2219604053a4d05b98e90d6a335511cc01806436ec4886b1028/mkdocs_autorefs-1.2.0-py3-none-any.whl", hash = "sha256:d588754ae89bd0ced0c70c06f58566a4ee43471eeeee5202427da7de9ef85a2f", size = 16522 }, +] + +[[package]] +name = "mkdocs-get-deps" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, + { name = "mergedeep" }, + { name = "platformdirs" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/f5/ed29cd50067784976f25ed0ed6fcd3c2ce9eb90650aa3b2796ddf7b6870b/mkdocs_get_deps-0.2.0.tar.gz", hash = "sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c", size = 10239 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/d4/029f984e8d3f3b6b726bd33cafc473b75e9e44c0f7e80a5b29abc466bdea/mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134", size = 9521 }, +] + +[[package]] +name = "mkdocs-material" +version = "9.5.34" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "babel" }, + { name = "colorama" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "mkdocs" }, + { name = "mkdocs-material-extensions" }, + { name = "paginate" }, + { name = "pygments" }, + { name = "pymdown-extensions" }, + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/79/9b1428841d1d9331561c9d2f22f572b8ceeb67432b383bb1f74f954d92c2/mkdocs_material-9.5.34.tar.gz", hash = "sha256:1e60ddf716cfb5679dfd65900b8a25d277064ed82d9a53cd5190e3f894df7840", size = 3975085 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/42/d6cc455d37979c5617cab5aac49bd15e07317f7a61a80ce3c49ce23c779f/mkdocs_material-9.5.34-py3-none-any.whl", hash = "sha256:54caa8be708de2b75167fd4d3b9f3d949579294f49cb242515d4653dbee9227e", size = 8668442 }, +] + +[[package]] +name = "mkdocs-material-extensions" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/79/9b/9b4c96d6593b2a541e1cb8b34899a6d021d208bb357042823d4d2cabdbe7/mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443", size = 11847 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728 }, +] + +[[package]] +name = "mkdocstrings" +version = "0.26.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe" }, + { name = "mkdocs" }, + { name = "mkdocs-autorefs" }, + { name = "platformdirs" }, + { name = "pymdown-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/bf/170ff04de72227f715d67da32950c7b8434449f3805b2ec3dd1085db4d7c/mkdocstrings-0.26.1.tar.gz", hash = "sha256:bb8b8854d6713d5348ad05b069a09f3b79edbc6a0f33a34c6821141adb03fe33", size = 92677 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/cc/8ba127aaee5d1e9046b0d33fa5b3d17da95a9d705d44902792e0569257fd/mkdocstrings-0.26.1-py3-none-any.whl", hash = "sha256:29738bfb72b4608e8e55cc50fb8a54f325dc7ebd2014e4e3881a49892d5983cf", size = 29643 }, +] + +[[package]] +name = "mkdocstrings-python" +version = "1.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "griffe" }, + { name = "mkdocs-autorefs" }, + { name = "mkdocstrings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/ba/534c934cd0a809f51c91332d6ed278782ee4126b8ba8db02c2003f162b47/mkdocstrings_python-1.11.1.tar.gz", hash = "sha256:8824b115c5359304ab0b5378a91f6202324a849e1da907a3485b59208b797322", size = 166890 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/f2/2a2c48fda645ac6bbe73bcc974587a579092b6868e6ff8bc6d177f4db38a/mkdocstrings_python-1.11.1-py3-none-any.whl", hash = "sha256:a21a1c05acef129a618517bb5aae3e33114f569b11588b1e7af3e9d4061a71af", size = 109297 }, +] + [[package]] name = "mkl" version = "2021.4.0" @@ -940,14 +1106,22 @@ wheels = [ ] [[package]] -name = "pdoc3" -version = "0.11.1" +name = "paginate" +version = "0.5.7" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mako" }, - { name = "markdown" }, +sdist = { url = "https://files.pythonhosted.org/packages/ec/46/68dde5b6bc00c1296ec6466ab27dddede6aec9af1b99090e1107091b3b84/paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945", size = 19252 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/96/04b8e52da071d28f5e21a805b19cb9390aa17a47462ac87f5e2696b9566d/paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591", size = 13746 }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/a3/141d32ae6abfdd17f0314a3f6b93ce83becefc7f0f71f5902b6b8b993cd7/pdoc3-0.11.1.tar.gz", hash = "sha256:20bf2aad8110892573fd350b1bf95f6612b53b55e29b0fc143b5f2b5bcbabfb6", size = 97928 } [[package]] name = "pillow" @@ -1026,6 +1200,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/37/ae/2dbfc38cc4fd14aceea14bc440d5151b21f64c4c3ba3f6f4191610b7ee5d/pillow-10.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cfdd747216947628af7b259d274771d84db2268ca062dd5faf373639d00113a3", size = 2554652 }, ] +[[package]] +name = "platformdirs" +version = "4.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/a0/d7cab8409cdc7d39b037c85ac46d92434fb6595432e069251b38e5c8dd0e/platformdirs-4.3.2.tar.gz", hash = "sha256:9e5e27a08aa095dd127b9f2e764d74254f482fef22b0970773bfba79d091ab8c", size = 21276 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/8b/d497999c4017b80678017ddce745cf675489c110681ad3c84a55eddfd3e7/platformdirs-4.3.2-py3-none-any.whl", hash = "sha256:eb1c8582560b34ed4ba105009a4badf7f6f85768b30126f351328507b2beb617", size = 18417 }, +] + [[package]] name = "pluggy" version = "1.5.0" @@ -1035,6 +1218,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 }, ] +[[package]] +name = "pygments" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/62/8336eff65bcbc8e4cb5d05b55faf041285951b6e80f33e2bff2024788f31/pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199", size = 4891905 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a", size = 1205513 }, +] + +[[package]] +name = "pymdown-extensions" +version = "10.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/d3/fb86beeaa4416f73a28a5e8d440976b7cada2b2d7b5e715b2bd849d4de32/pymdown_extensions-10.9.tar.gz", hash = "sha256:6ff740bcd99ec4172a938970d42b96128bdc9d4b9bcad72494f29921dc69b753", size = 812128 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/41/18b5dc5e97ec3ff1c2f51d372e570a9fbe231f1124dcc36dbc6b47f93058/pymdown_extensions-10.9-py3-none-any.whl", hash = "sha256:d323f7e90d83c86113ee78f3fe62fc9dee5f56b54d912660703ea1816fed5626", size = 250954 }, +] + [[package]] name = "pyparsing" version = "3.1.2" @@ -1098,6 +1303,156 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, ] +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199 }, + { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758 }, + { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463 }, + { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280 }, + { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239 }, + { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802 }, + { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527 }, + { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052 }, + { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774 }, + { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612 }, + { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040 }, + { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829 }, + { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167 }, + { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952 }, + { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301 }, + { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638 }, + { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850 }, + { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980 }, + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873 }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302 }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154 }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223 }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542 }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164 }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611 }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591 }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309 }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679 }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428 }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361 }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523 }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660 }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597 }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527 }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446 }, + { url = "https://files.pythonhosted.org/packages/65/d8/b7a1db13636d7fb7d4ff431593c510c8b8fca920ade06ca8ef20015493c5/PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d", size = 184777 }, + { url = "https://files.pythonhosted.org/packages/0a/02/6ec546cd45143fdf9840b2c6be8d875116a64076218b61d68e12548e5839/PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f", size = 172318 }, + { url = "https://files.pythonhosted.org/packages/0e/9a/8cc68be846c972bda34f6c2a93abb644fb2476f4dcc924d52175786932c9/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290", size = 720891 }, + { url = "https://files.pythonhosted.org/packages/e9/6c/6e1b7f40181bc4805e2e07f4abc10a88ce4648e7e95ff1abe4ae4014a9b2/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12", size = 722614 }, + { url = "https://files.pythonhosted.org/packages/3d/32/e7bd8535d22ea2874cef6a81021ba019474ace0d13a4819c2a4bce79bd6a/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19", size = 737360 }, + { url = "https://files.pythonhosted.org/packages/d7/12/7322c1e30b9be969670b672573d45479edef72c9a0deac3bb2868f5d7469/PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e", size = 699006 }, + { url = "https://files.pythonhosted.org/packages/82/72/04fcad41ca56491995076630c3ec1e834be241664c0c09a64c9a2589b507/PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725", size = 723577 }, + { url = "https://files.pythonhosted.org/packages/ed/5e/46168b1f2757f1fcd442bc3029cd8767d88a98c9c05770d8b420948743bb/PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631", size = 144593 }, + { url = "https://files.pythonhosted.org/packages/19/87/5124b1c1f2412bb95c59ec481eaf936cd32f0fe2a7b16b97b81c4c017a6a/PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8", size = 162312 }, +] + +[[package]] +name = "pyyaml-env-tag" +version = "0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/8e/da1c6c58f751b70f8ceb1eb25bc25d524e8f14fe16edcce3f4e3ba08629c/pyyaml_env_tag-0.1.tar.gz", hash = "sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb", size = 5631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/66/bbb1dd374f5c870f59c5bb1db0e18cbe7fa739415a24cbd95b2d1f5ae0c4/pyyaml_env_tag-0.1-py3-none-any.whl", hash = "sha256:af31106dec8a4d68c60207c1886031cbf839b68aa7abccdb19868200532c2069", size = 3911 }, +] + +[[package]] +name = "regex" +version = "2024.9.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/38/148df33b4dbca3bd069b963acab5e0fa1a9dbd6820f8c322d0dd6faeff96/regex-2024.9.11.tar.gz", hash = "sha256:6c188c307e8433bcb63dc1915022deb553b4203a70722fc542c363bf120a01fd", size = 399403 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/12/497bd6599ce8a239ade68678132296aec5ee25ebea45fc8ba91aa60fceec/regex-2024.9.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1494fa8725c285a81d01dc8c06b55287a1ee5e0e382d8413adc0a9197aac6408", size = 482488 }, + { url = "https://files.pythonhosted.org/packages/c1/24/595ddb9bec2a9b151cdaf9565b0c9f3da9f0cb1dca6c158bc5175332ddf8/regex-2024.9.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e12c481ad92d129c78f13a2a3662317e46ee7ef96c94fd332e1c29131875b7d", size = 287443 }, + { url = "https://files.pythonhosted.org/packages/69/a8/b2fb45d9715b1469383a0da7968f8cacc2f83e9fbbcd6b8713752dd980a6/regex-2024.9.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:16e13a7929791ac1216afde26f712802e3df7bf0360b32e4914dca3ab8baeea5", size = 284561 }, + { url = "https://files.pythonhosted.org/packages/88/87/1ce4a5357216b19b7055e7d3b0efc75a6e426133bf1e7d094321df514257/regex-2024.9.11-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46989629904bad940bbec2106528140a218b4a36bb3042d8406980be1941429c", size = 783177 }, + { url = "https://files.pythonhosted.org/packages/3c/65/b9f002ab32f7b68e7d1dcabb67926f3f47325b8dbc22cc50b6a043e1d07c/regex-2024.9.11-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a906ed5e47a0ce5f04b2c981af1c9acf9e8696066900bf03b9d7879a6f679fc8", size = 823193 }, + { url = "https://files.pythonhosted.org/packages/22/91/8339dd3abce101204d246e31bc26cdd7ec07c9f91598472459a3a902aa41/regex-2024.9.11-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9a091b0550b3b0207784a7d6d0f1a00d1d1c8a11699c1a4d93db3fbefc3ad35", size = 809950 }, + { url = "https://files.pythonhosted.org/packages/cb/19/556638aa11c2ec9968a1da998f07f27ec0abb9bf3c647d7c7985ca0b8eea/regex-2024.9.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ddcd9a179c0a6fa8add279a4444015acddcd7f232a49071ae57fa6e278f1f71", size = 782661 }, + { url = "https://files.pythonhosted.org/packages/d1/e9/7a5bc4c6ef8d9cd2bdd83a667888fc35320da96a4cc4da5fa084330f53db/regex-2024.9.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6b41e1adc61fa347662b09398e31ad446afadff932a24807d3ceb955ed865cc8", size = 772348 }, + { url = "https://files.pythonhosted.org/packages/f1/0b/29f2105bfac3ed08e704914c38e93b07c784a6655f8a015297ee7173e95b/regex-2024.9.11-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ced479f601cd2f8ca1fd7b23925a7e0ad512a56d6e9476f79b8f381d9d37090a", size = 697460 }, + { url = "https://files.pythonhosted.org/packages/71/3a/52ff61054d15a4722605f5872ad03962b319a04c1ebaebe570b8b9b7dde1/regex-2024.9.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:635a1d96665f84b292e401c3d62775851aedc31d4f8784117b3c68c4fcd4118d", size = 769151 }, + { url = "https://files.pythonhosted.org/packages/97/07/37e460ab5ca84be8e1e197c3b526c5c86993dcc9e13cbc805c35fc2463c1/regex-2024.9.11-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c0256beda696edcf7d97ef16b2a33a8e5a875affd6fa6567b54f7c577b30a137", size = 777478 }, + { url = "https://files.pythonhosted.org/packages/65/7b/953075723dd5ab00780043ac2f9de667306ff9e2a85332975e9f19279174/regex-2024.9.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:3ce4f1185db3fbde8ed8aa223fc9620f276c58de8b0d4f8cc86fd1360829edb6", size = 845373 }, + { url = "https://files.pythonhosted.org/packages/40/b8/3e9484c6230b8b6e8f816ab7c9a080e631124991a4ae2c27a81631777db0/regex-2024.9.11-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:09d77559e80dcc9d24570da3745ab859a9cf91953062e4ab126ba9d5993688ca", size = 845369 }, + { url = "https://files.pythonhosted.org/packages/b7/99/38434984d912edbd2e1969d116257e869578f67461bd7462b894c45ed874/regex-2024.9.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a22ccefd4db3f12b526eccb129390942fe874a3a9fdbdd24cf55773a1faab1a", size = 773935 }, + { url = "https://files.pythonhosted.org/packages/ab/67/43174d2b46fa947b7b9dfe56b6c8a8a76d44223f35b1d64645a732fd1d6f/regex-2024.9.11-cp310-cp310-win32.whl", hash = "sha256:f745ec09bc1b0bd15cfc73df6fa4f726dcc26bb16c23a03f9e3367d357eeedd0", size = 261624 }, + { url = "https://files.pythonhosted.org/packages/c4/2a/4f9c47d9395b6aff24874c761d8d620c0232f97c43ef3cf668c8b355e7a7/regex-2024.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:01c2acb51f8a7d6494c8c5eafe3d8e06d76563d8a8a4643b37e9b2dd8a2ff623", size = 274020 }, + { url = "https://files.pythonhosted.org/packages/86/a1/d526b7b6095a0019aa360948c143aacfeb029919c898701ce7763bbe4c15/regex-2024.9.11-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2cce2449e5927a0bf084d346da6cd5eb016b2beca10d0013ab50e3c226ffc0df", size = 482483 }, + { url = "https://files.pythonhosted.org/packages/32/d9/bfdd153179867c275719e381e1e8e84a97bd186740456a0dcb3e7125c205/regex-2024.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b37fa423beefa44919e009745ccbf353d8c981516e807995b2bd11c2c77d268", size = 287442 }, + { url = "https://files.pythonhosted.org/packages/33/c4/60f3370735135e3a8d673ddcdb2507a8560d0e759e1398d366e43d000253/regex-2024.9.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64ce2799bd75039b480cc0360907c4fb2f50022f030bf9e7a8705b636e408fad", size = 284561 }, + { url = "https://files.pythonhosted.org/packages/b1/51/91a5ebdff17f9ec4973cb0aa9d37635efec1c6868654bbc25d1543aca4ec/regex-2024.9.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4cc92bb6db56ab0c1cbd17294e14f5e9224f0cc6521167ef388332604e92679", size = 791779 }, + { url = "https://files.pythonhosted.org/packages/07/4a/022c5e6f0891a90cd7eb3d664d6c58ce2aba48bff107b00013f3d6167069/regex-2024.9.11-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d05ac6fa06959c4172eccd99a222e1fbf17b5670c4d596cb1e5cde99600674c4", size = 832605 }, + { url = "https://files.pythonhosted.org/packages/ac/1c/3793990c8c83ca04e018151ddda83b83ecc41d89964f0f17749f027fc44d/regex-2024.9.11-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040562757795eeea356394a7fb13076ad4f99d3c62ab0f8bdfb21f99a1f85664", size = 818556 }, + { url = "https://files.pythonhosted.org/packages/e9/5c/8b385afbfacb853730682c57be56225f9fe275c5bf02ac1fc88edbff316d/regex-2024.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6113c008a7780792efc80f9dfe10ba0cd043cbf8dc9a76ef757850f51b4edc50", size = 792808 }, + { url = "https://files.pythonhosted.org/packages/9b/8b/a4723a838b53c771e9240951adde6af58c829fb6a6a28f554e8131f53839/regex-2024.9.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e5fb5f77c8745a60105403a774fe2c1759b71d3e7b4ca237a5e67ad066c7199", size = 781115 }, + { url = "https://files.pythonhosted.org/packages/83/5f/031a04b6017033d65b261259c09043c06f4ef2d4eac841d0649d76d69541/regex-2024.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:54d9ff35d4515debf14bc27f1e3b38bfc453eff3220f5bce159642fa762fe5d4", size = 778155 }, + { url = "https://files.pythonhosted.org/packages/fd/cd/4660756070b03ce4a66663a43f6c6e7ebc2266cc6b4c586c167917185eb4/regex-2024.9.11-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df5cbb1fbc74a8305b6065d4ade43b993be03dbe0f8b30032cced0d7740994bd", size = 784614 }, + { url = "https://files.pythonhosted.org/packages/93/8d/65b9bea7df120a7be8337c415b6d256ba786cbc9107cebba3bf8ff09da99/regex-2024.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7fb89ee5d106e4a7a51bce305ac4efb981536301895f7bdcf93ec92ae0d91c7f", size = 853744 }, + { url = "https://files.pythonhosted.org/packages/96/a7/fba1eae75eb53a704475baf11bd44b3e6ccb95b316955027eb7748f24ef8/regex-2024.9.11-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a738b937d512b30bf75995c0159c0ddf9eec0775c9d72ac0202076c72f24aa96", size = 855890 }, + { url = "https://files.pythonhosted.org/packages/45/14/d864b2db80a1a3358534392373e8a281d95b28c29c87d8548aed58813910/regex-2024.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e28f9faeb14b6f23ac55bfbbfd3643f5c7c18ede093977f1df249f73fd22c7b1", size = 781887 }, + { url = "https://files.pythonhosted.org/packages/4d/a9/bfb29b3de3eb11dc9b412603437023b8e6c02fb4e11311863d9bf62c403a/regex-2024.9.11-cp311-cp311-win32.whl", hash = "sha256:18e707ce6c92d7282dfce370cd205098384b8ee21544e7cb29b8aab955b66fa9", size = 261644 }, + { url = "https://files.pythonhosted.org/packages/c7/ab/1ad2511cf6a208fde57fafe49829cab8ca018128ab0d0b48973d8218634a/regex-2024.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:313ea15e5ff2a8cbbad96ccef6be638393041b0a7863183c2d31e0c6116688cf", size = 274033 }, + { url = "https://files.pythonhosted.org/packages/6e/92/407531450762bed778eedbde04407f68cbd75d13cee96c6f8d6903d9c6c1/regex-2024.9.11-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b0d0a6c64fcc4ef9c69bd5b3b3626cc3776520a1637d8abaa62b9edc147a58f7", size = 483590 }, + { url = "https://files.pythonhosted.org/packages/8e/a2/048acbc5ae1f615adc6cba36cc45734e679b5f1e4e58c3c77f0ed611d4e2/regex-2024.9.11-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:49b0e06786ea663f933f3710a51e9385ce0cba0ea56b67107fd841a55d56a231", size = 288175 }, + { url = "https://files.pythonhosted.org/packages/8a/ea/909d8620329ab710dfaf7b4adee41242ab7c9b95ea8d838e9bfe76244259/regex-2024.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5b513b6997a0b2f10e4fd3a1313568e373926e8c252bd76c960f96fd039cd28d", size = 284749 }, + { url = "https://files.pythonhosted.org/packages/ca/fa/521eb683b916389b4975337873e66954e0f6d8f91bd5774164a57b503185/regex-2024.9.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee439691d8c23e76f9802c42a95cfeebf9d47cf4ffd06f18489122dbb0a7ad64", size = 795181 }, + { url = "https://files.pythonhosted.org/packages/28/db/63047feddc3280cc242f9c74f7aeddc6ee662b1835f00046f57d5630c827/regex-2024.9.11-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8f877c89719d759e52783f7fe6e1c67121076b87b40542966c02de5503ace42", size = 835842 }, + { url = "https://files.pythonhosted.org/packages/e3/94/86adc259ff8ec26edf35fcca7e334566c1805c7493b192cb09679f9c3dee/regex-2024.9.11-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23b30c62d0f16827f2ae9f2bb87619bc4fba2044911e2e6c2eb1af0161cdb766", size = 823533 }, + { url = "https://files.pythonhosted.org/packages/29/52/84662b6636061277cb857f658518aa7db6672bc6d1a3f503ccd5aefc581e/regex-2024.9.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85ab7824093d8f10d44330fe1e6493f756f252d145323dd17ab6b48733ff6c0a", size = 797037 }, + { url = "https://files.pythonhosted.org/packages/c3/2a/cd4675dd987e4a7505f0364a958bc41f3b84942de9efaad0ef9a2646681c/regex-2024.9.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dee5b4810a89447151999428fe096977346cf2f29f4d5e29609d2e19e0199c9", size = 784106 }, + { url = "https://files.pythonhosted.org/packages/6f/75/3ea7ec29de0bbf42f21f812f48781d41e627d57a634f3f23947c9a46e303/regex-2024.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98eeee2f2e63edae2181c886d7911ce502e1292794f4c5ee71e60e23e8d26b5d", size = 782468 }, + { url = "https://files.pythonhosted.org/packages/d3/67/15519d69b52c252b270e679cb578e22e0c02b8dd4e361f2b04efcc7f2335/regex-2024.9.11-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:57fdd2e0b2694ce6fc2e5ccf189789c3e2962916fb38779d3e3521ff8fe7a822", size = 790324 }, + { url = "https://files.pythonhosted.org/packages/9c/71/eff77d3fe7ba08ab0672920059ec30d63fa7e41aa0fb61c562726e9bd721/regex-2024.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d552c78411f60b1fdaafd117a1fca2f02e562e309223b9d44b7de8be451ec5e0", size = 860214 }, + { url = "https://files.pythonhosted.org/packages/81/11/e1bdf84a72372e56f1ea4b833dd583b822a23138a616ace7ab57a0e11556/regex-2024.9.11-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a0b2b80321c2ed3fcf0385ec9e51a12253c50f146fddb2abbb10f033fe3d049a", size = 859420 }, + { url = "https://files.pythonhosted.org/packages/ea/75/9753e9dcebfa7c3645563ef5c8a58f3a47e799c872165f37c55737dadd3e/regex-2024.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:18406efb2f5a0e57e3a5881cd9354c1512d3bb4f5c45d96d110a66114d84d23a", size = 787333 }, + { url = "https://files.pythonhosted.org/packages/bc/4e/ba1cbca93141f7416624b3ae63573e785d4bc1834c8be44a8f0747919eca/regex-2024.9.11-cp312-cp312-win32.whl", hash = "sha256:e464b467f1588e2c42d26814231edecbcfe77f5ac414d92cbf4e7b55b2c2a776", size = 262058 }, + { url = "https://files.pythonhosted.org/packages/6e/16/efc5f194778bf43e5888209e5cec4b258005d37c613b67ae137df3b89c53/regex-2024.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:9e8719792ca63c6b8340380352c24dcb8cd7ec49dae36e963742a275dfae6009", size = 273526 }, + { url = "https://files.pythonhosted.org/packages/93/0a/d1c6b9af1ff1e36832fe38d74d5c5bab913f2bdcbbd6bc0e7f3ce8b2f577/regex-2024.9.11-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c157bb447303070f256e084668b702073db99bbb61d44f85d811025fcf38f784", size = 483376 }, + { url = "https://files.pythonhosted.org/packages/a4/42/5910a050c105d7f750a72dcb49c30220c3ae4e2654e54aaaa0e9bc0584cb/regex-2024.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4db21ece84dfeefc5d8a3863f101995de646c6cb0536952c321a2650aa202c36", size = 288112 }, + { url = "https://files.pythonhosted.org/packages/8d/56/0c262aff0e9224fa7ffce47b5458d373f4d3e3ff84e99b5ff0cb15e0b5b2/regex-2024.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:220e92a30b426daf23bb67a7962900ed4613589bab80382be09b48896d211e92", size = 284608 }, + { url = "https://files.pythonhosted.org/packages/b9/54/9fe8f9aec5007bbbbce28ba3d2e3eaca425f95387b7d1e84f0d137d25237/regex-2024.9.11-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1ae19e64c14c7ec1995f40bd932448713d3c73509e82d8cd7744dc00e29e86", size = 795337 }, + { url = "https://files.pythonhosted.org/packages/b2/e7/6b2f642c3cded271c4f16cc4daa7231be544d30fe2b168e0223724b49a61/regex-2024.9.11-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f47cd43a5bfa48f86925fe26fbdd0a488ff15b62468abb5d2a1e092a4fb10e85", size = 835848 }, + { url = "https://files.pythonhosted.org/packages/cd/9e/187363bdf5d8c0e4662117b92aa32bf52f8f09620ae93abc7537d96d3311/regex-2024.9.11-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9d4a76b96f398697fe01117093613166e6aa8195d63f1b4ec3f21ab637632963", size = 823503 }, + { url = "https://files.pythonhosted.org/packages/f8/10/601303b8ee93589f879664b0cfd3127949ff32b17f9b6c490fb201106c4d/regex-2024.9.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ea51dcc0835eea2ea31d66456210a4e01a076d820e9039b04ae8d17ac11dee6", size = 797049 }, + { url = "https://files.pythonhosted.org/packages/ef/1c/ea200f61ce9f341763f2717ab4daebe4422d83e9fd4ac5e33435fd3a148d/regex-2024.9.11-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7aaa315101c6567a9a45d2839322c51c8d6e81f67683d529512f5bcfb99c802", size = 784144 }, + { url = "https://files.pythonhosted.org/packages/d8/5c/d2429be49ef3292def7688401d3deb11702c13dcaecdc71d2b407421275b/regex-2024.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c57d08ad67aba97af57a7263c2d9006d5c404d721c5f7542f077f109ec2a4a29", size = 782483 }, + { url = "https://files.pythonhosted.org/packages/12/d9/cbc30f2ff7164f3b26a7760f87c54bf8b2faed286f60efd80350a51c5b99/regex-2024.9.11-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8404bf61298bb6f8224bb9176c1424548ee1181130818fcd2cbffddc768bed8", size = 790320 }, + { url = "https://files.pythonhosted.org/packages/19/1d/43ed03a236313639da5a45e61bc553c8d41e925bcf29b0f8ecff0c2c3f25/regex-2024.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dd4490a33eb909ef5078ab20f5f000087afa2a4daa27b4c072ccb3cb3050ad84", size = 860435 }, + { url = "https://files.pythonhosted.org/packages/34/4f/5d04da61c7c56e785058a46349f7285ae3ebc0726c6ea7c5c70600a52233/regex-2024.9.11-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:eee9130eaad130649fd73e5cd92f60e55708952260ede70da64de420cdcad554", size = 859571 }, + { url = "https://files.pythonhosted.org/packages/12/7f/8398c8155a3c70703a8e91c29532558186558e1aea44144b382faa2a6f7a/regex-2024.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6a2644a93da36c784e546de579ec1806bfd2763ef47babc1b03d765fe560c9f8", size = 787398 }, + { url = "https://files.pythonhosted.org/packages/58/3a/f5903977647a9a7e46d5535e9e96c194304aeeca7501240509bde2f9e17f/regex-2024.9.11-cp313-cp313-win32.whl", hash = "sha256:e997fd30430c57138adc06bba4c7c2968fb13d101e57dd5bb9355bf8ce3fa7e8", size = 262035 }, + { url = "https://files.pythonhosted.org/packages/ff/80/51ba3a4b7482f6011095b3a036e07374f64de180b7d870b704ed22509002/regex-2024.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:042c55879cfeb21a8adacc84ea347721d3d83a159da6acdf1116859e2427c43f", size = 273510 }, + { url = "https://files.pythonhosted.org/packages/a1/aa/e31baf8482ad690ccb3cdf20d1963a01e98d137e4d9ee493dbb0fa8ba2c6/regex-2024.9.11-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:07f45f287469039ffc2c53caf6803cd506eb5f5f637f1d4acb37a738f71dd066", size = 482489 }, + { url = "https://files.pythonhosted.org/packages/a1/b5/449c2f14fc20dc42ef9729469fcff42809393470f021ed6c6fcf5f3d3297/regex-2024.9.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4838e24ee015101d9f901988001038f7f0d90dc0c3b115541a1365fb439add62", size = 287440 }, + { url = "https://files.pythonhosted.org/packages/3f/36/4b60a0c2e4cc6ecb2651be828117a31f42fae55a51a484a8071729df56a6/regex-2024.9.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6edd623bae6a737f10ce853ea076f56f507fd7726bee96a41ee3d68d347e4d16", size = 284566 }, + { url = "https://files.pythonhosted.org/packages/b4/21/feaa5b0d3e5e3bad659cd7d640e6b76cc0719504dbd9bc8f67cfa21bde82/regex-2024.9.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c69ada171c2d0e97a4b5aa78fbb835e0ffbb6b13fc5da968c09811346564f0d3", size = 782747 }, + { url = "https://files.pythonhosted.org/packages/bb/89/93516f0aa3e8a9366df2cf79bb0290abdc7dbe5dd27373d9bea0978b7ba6/regex-2024.9.11-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02087ea0a03b4af1ed6ebab2c54d7118127fee8d71b26398e8e4b05b78963199", size = 822700 }, + { url = "https://files.pythonhosted.org/packages/d5/e7/79c04ccb81cee2831d9d4499274919b9153c1741ce8b3421d69cb0032f1b/regex-2024.9.11-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:69dee6a020693d12a3cf892aba4808fe168d2a4cef368eb9bf74f5398bfd4ee8", size = 809327 }, + { url = "https://files.pythonhosted.org/packages/01/e6/a7256c99c312b68f01cfd4f8eae6e770906fffb3832ecb66f35ca5b86b96/regex-2024.9.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:297f54910247508e6e5cae669f2bc308985c60540a4edd1c77203ef19bfa63ca", size = 781970 }, + { url = "https://files.pythonhosted.org/packages/18/c4/29e8b6ff2208775858b5d4a2caa6428d40b5fade95aee426de7e42ffff39/regex-2024.9.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecea58b43a67b1b79805f1a0255730edaf5191ecef84dbc4cc85eb30bc8b63b9", size = 771885 }, + { url = "https://files.pythonhosted.org/packages/95/78/7acd8882ac335f1f5ae1756417739fda3053e0bcacea8716ae4a04e74553/regex-2024.9.11-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eab4bb380f15e189d1313195b062a6aa908f5bd687a0ceccd47c8211e9cf0d4a", size = 696978 }, + { url = "https://files.pythonhosted.org/packages/cb/d2/1d44f9b4a3d33ff5773fd79bea53e992d00f81e0af6f1f4e2efac1e4d897/regex-2024.9.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0cbff728659ce4bbf4c30b2a1be040faafaa9eca6ecde40aaff86f7889f4ab39", size = 768655 }, + { url = "https://files.pythonhosted.org/packages/79/ba/92ef9d3b8f59cb3df9febef07098dfb4a43c3bdcf35b1084c2009b0a93bf/regex-2024.9.11-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:54c4a097b8bc5bb0dfc83ae498061d53ad7b5762e00f4adaa23bee22b012e6ba", size = 776922 }, + { url = "https://files.pythonhosted.org/packages/16/71/d964c0c9d447f04bbe6ab5eafd220208e7d52b9608e452e6fcad553b38e0/regex-2024.9.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:73d6d2f64f4d894c96626a75578b0bf7d9e56dcda8c3d037a2118fdfe9b1c664", size = 845014 }, + { url = "https://files.pythonhosted.org/packages/83/cb/a378cdc2468782eefefa50183bbeabc3357fb588d4109d845f0a56e68713/regex-2024.9.11-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:e53b5fbab5d675aec9f0c501274c467c0f9a5d23696cfc94247e1fb56501ed89", size = 844916 }, + { url = "https://files.pythonhosted.org/packages/b9/f0/82ea1565a6639270cfe96263002b3d91084a1db5048d9b6084f83bd5972d/regex-2024.9.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0ffbcf9221e04502fc35e54d1ce9567541979c3fdfb93d2c554f0ca583a19b35", size = 773409 }, + { url = "https://files.pythonhosted.org/packages/97/9e/0400d742b9647b4940609a96d550de89e4e89c85f6a370796dab25b5979c/regex-2024.9.11-cp39-cp39-win32.whl", hash = "sha256:e4c22e1ac1f1ec1e09f72e6c44d8f2244173db7eb9629cc3a346a8d7ccc31142", size = 261680 }, + { url = "https://files.pythonhosted.org/packages/b6/f1/aef1112652ac7b3922d2c129f8325a4fd286b66691127dd99f380f8ede19/regex-2024.9.11-cp39-cp39-win_amd64.whl", hash = "sha256:faa3c142464efec496967359ca99696c896c591c56c53506bac1ad465f66e919", size = 274066 }, +] + [[package]] name = "requests" version = "2.32.3" @@ -1392,6 +1747,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/1c/89ffc63a9605b583d5df2be791a27bc1a42b7c32bab68d3c8f2f73a98cd4/urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", size = 121444 }, ] +[[package]] +name = "watchdog" +version = "5.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/5e/95dcd86d8339fcf76385f7fad5e49cbfd989b6c6199127121c9587febc65/watchdog-5.0.2.tar.gz", hash = "sha256:dcebf7e475001d2cdeb020be630dc5b687e9acdd60d16fea6bb4508e7b94cf76", size = 127779 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/19/c5b0f64269d396dbc9f06d4b7fa8400c0282143640ebc8cbad84553ee4ee/watchdog-5.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d961f4123bb3c447d9fcdcb67e1530c366f10ab3a0c7d1c0c9943050936d4877", size = 96274 }, + { url = "https://files.pythonhosted.org/packages/5e/5e/62adbcf4d96a533d71dbd951a3c101019989c8ce8796e267d6509ba12138/watchdog-5.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72990192cb63872c47d5e5fefe230a401b87fd59d257ee577d61c9e5564c62e5", size = 88275 }, + { url = "https://files.pythonhosted.org/packages/d0/16/5b36358158b7debcae7b62fe9b6d9874c60e445b37b1e51b7c5d00c6572b/watchdog-5.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bec703ad90b35a848e05e1b40bf0050da7ca28ead7ac4be724ae5ac2653a1a0", size = 88911 }, + { url = "https://files.pythonhosted.org/packages/80/52/51046f428e813270cd959bee9d2343f103c10adf10e957f69d6710a38ab8/watchdog-5.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:dae7a1879918f6544201d33666909b040a46421054a50e0f773e0d870ed7438d", size = 96276 }, + { url = "https://files.pythonhosted.org/packages/b3/8e/0e5671f3950fd2049bbeb8c965cb53e143bfd72869e5e4c60dda572121cd/watchdog-5.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c4a440f725f3b99133de610bfec93d570b13826f89616377715b9cd60424db6e", size = 88269 }, + { url = "https://files.pythonhosted.org/packages/b5/34/9c436ec85f7234b468e49380f57cc784b4e22f058febe17221f25ce85c4b/watchdog-5.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8b2918c19e0d48f5f20df458c84692e2a054f02d9df25e6c3c930063eca64c1", size = 88914 }, + { url = "https://files.pythonhosted.org/packages/ef/41/fe19a56aa8ea7e453311f2b4fd2bfb172d21bd72ef6ae0fd40c304c74edf/watchdog-5.0.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:aa9cd6e24126d4afb3752a3e70fce39f92d0e1a58a236ddf6ee823ff7dba28ee", size = 96365 }, + { url = "https://files.pythonhosted.org/packages/cc/02/86d631595ec1c5678e23e9359741d2dea460be0712b41a243281b37e90ba/watchdog-5.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f627c5bf5759fdd90195b0c0431f99cff4867d212a67b384442c51136a098ed7", size = 88330 }, + { url = "https://files.pythonhosted.org/packages/d8/a7/5c57f05def91ff11528f0aa0d4c23efc99fa064ec69c262fedc6c9885697/watchdog-5.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7594a6d32cda2b49df3fd9abf9b37c8d2f3eab5df45c24056b4a671ac661619", size = 88935 }, + { url = "https://files.pythonhosted.org/packages/80/1a/a681c0093eea33b18a7348b398302628ab96647f59eaf06a5a047e8a1f39/watchdog-5.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba32efcccfe2c58f4d01115440d1672b4eb26cdd6fc5b5818f1fb41f7c3e1889", size = 96362 }, + { url = "https://files.pythonhosted.org/packages/c4/aa/0c827bd35716d91b5a4a2a6c5ca7638d936e6055dec8ce85414383ab887f/watchdog-5.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:963f7c4c91e3f51c998eeff1b3fb24a52a8a34da4f956e470f4b068bb47b78ee", size = 88336 }, + { url = "https://files.pythonhosted.org/packages/6e/ba/da13d47dacc84bfab52310e74f954eb440c5cdee11ff8786228f17343a3d/watchdog-5.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8c47150aa12f775e22efff1eee9f0f6beee542a7aa1a985c271b1997d340184f", size = 88938 }, + { url = "https://files.pythonhosted.org/packages/e3/db/83ae143fced93f7e5962d2aa6f938d3986a5931c6e68dafea65d40b40fd3/watchdog-5.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:14dd4ed023d79d1f670aa659f449bcd2733c33a35c8ffd88689d9d243885198b", size = 96273 }, + { url = "https://files.pythonhosted.org/packages/fc/c2/48b61c5668e8a0692a823b8c0a965b39c80768d82cd3c0d2f5b17ead6e43/watchdog-5.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b84bff0391ad4abe25c2740c7aec0e3de316fdf7764007f41e248422a7760a7f", size = 88271 }, + { url = "https://files.pythonhosted.org/packages/ca/53/57e380b6b88dcbb47a6cad077abdc1fbdd12bf153f8e2ed8e48c5ffacbbd/watchdog-5.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e8d5ff39f0a9968952cce548e8e08f849141a4fcc1290b1c17c032ba697b9d7", size = 88910 }, + { url = "https://files.pythonhosted.org/packages/23/f1/dafce06a12fe2d61859aaceb81fbe3f3ed7907b81fcfa784416b1196dcfe/watchdog-5.0.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:fb223456db6e5f7bd9bbd5cd969f05aae82ae21acc00643b60d81c770abd402b", size = 87795 }, + { url = "https://files.pythonhosted.org/packages/7c/63/39a71aa9cea895885b3e644b573f1d05e00e368211efe76b9a63c7623512/watchdog-5.0.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9814adb768c23727a27792c77812cf4e2fd9853cd280eafa2bcfa62a99e8bd6e", size = 88269 }, + { url = "https://files.pythonhosted.org/packages/06/6d/866cacf6f17db488cbe117dd8e18712b4e316f16e61b9e6e104d4ce4b512/watchdog-5.0.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:901ee48c23f70193d1a7bc2d9ee297df66081dd5f46f0ca011be4f70dec80dab", size = 87794 }, + { url = "https://files.pythonhosted.org/packages/14/83/06ea29be9e7c6dfd1224c98e37fef2e20c63ceffa7fb71622bc82b55da1d/watchdog-5.0.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:638bcca3d5b1885c6ec47be67bf712b00a9ab3d4b22ec0881f4889ad870bc7e8", size = 88270 }, + { url = "https://files.pythonhosted.org/packages/5b/cb/c13dfc4714547c4a63f27a50d5d0bbda655ef06d93595c016822ff771032/watchdog-5.0.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5597c051587f8757798216f2485e85eac583c3b343e9aa09127a3a6f82c65ee8", size = 78960 }, + { url = "https://files.pythonhosted.org/packages/cb/ed/78acaa8e95e193a46925f7beeed45c29569d0ee572216df622bb0908abf3/watchdog-5.0.2-py3-none-manylinux2014_armv7l.whl", hash = "sha256:53ed1bf71fcb8475dd0ef4912ab139c294c87b903724b6f4a8bd98e026862e6d", size = 78960 }, + { url = "https://files.pythonhosted.org/packages/2f/54/30bde6279d2f77e6c2838a89e9975038bba4adbfb029f9b8e01cf2813199/watchdog-5.0.2-py3-none-manylinux2014_i686.whl", hash = "sha256:29e4a2607bd407d9552c502d38b45a05ec26a8e40cc7e94db9bb48f861fa5abc", size = 78958 }, + { url = "https://files.pythonhosted.org/packages/f4/db/886241c6d02f165fbf633b633dc5ceddc6c145fec3704828606743ddb663/watchdog-5.0.2-py3-none-manylinux2014_ppc64.whl", hash = "sha256:b6dc8f1d770a8280997e4beae7b9a75a33b268c59e033e72c8a10990097e5fde", size = 78957 }, + { url = "https://files.pythonhosted.org/packages/a9/74/c255a2146280adcb2d1b5ccb7580e71114b253f356a6c4ea748b0eb7a7b5/watchdog-5.0.2-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:d2ab34adc9bf1489452965cdb16a924e97d4452fcf88a50b21859068b50b5c3b", size = 78960 }, + { url = "https://files.pythonhosted.org/packages/8a/dc/4bdc31a35ffce526280c5a29b64b939624761f47e3fcdac34808589d0845/watchdog-5.0.2-py3-none-manylinux2014_s390x.whl", hash = "sha256:7d1aa7e4bb0f0c65a1a91ba37c10e19dabf7eaaa282c5787e51371f090748f4b", size = 78959 }, + { url = "https://files.pythonhosted.org/packages/9d/53/e71b01aa5737a21664b731de5f91c5b0721ff64d237e43efc56a99254fa1/watchdog-5.0.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:726eef8f8c634ac6584f86c9c53353a010d9f311f6c15a034f3800a7a891d941", size = 78959 }, + { url = "https://files.pythonhosted.org/packages/5d/0e/c37862900200436a554a4c411645f29887fe3fb4d4e465fbedcf1e0e383a/watchdog-5.0.2-py3-none-win32.whl", hash = "sha256:bda40c57115684d0216556671875e008279dea2dc00fcd3dde126ac8e0d7a2fb", size = 78947 }, + { url = "https://files.pythonhosted.org/packages/8f/ab/f1a3791be609e18596ce6a52c00274f1b244340b87379eb78c4df15f6b2b/watchdog-5.0.2-py3-none-win_amd64.whl", hash = "sha256:d010be060c996db725fbce7e3ef14687cdcc76f4ca0e4339a68cc4532c382a73", size = 78950 }, + { url = "https://files.pythonhosted.org/packages/53/99/f5065334d157518ec8c707aa790c93d639fac582be4f7caec5db8c6fa089/watchdog-5.0.2-py3-none-win_ia64.whl", hash = "sha256:3960136b2b619510569b90f0cd96408591d6c251a75c97690f4553ca88889769", size = 78948 }, +] + [[package]] name = "zipp" version = "3.20.0" From 762c0446acbbd26640be069374f545ba78d4015e Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Fri, 13 Sep 2024 16:18:10 -0400 Subject: [PATCH 05/10] Add deploy action --- .github/workflows/deploy-mkdocs.yml | 29 ++++++++++++++++ template/config.mako | 54 ----------------------------- update_docs.sh | 5 --- 3 files changed, 29 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/deploy-mkdocs.yml delete mode 100644 template/config.mako delete mode 100644 update_docs.sh diff --git a/.github/workflows/deploy-mkdocs.yml b/.github/workflows/deploy-mkdocs.yml new file mode 100644 index 00000000..4529cb0b --- /dev/null +++ b/.github/workflows/deploy-mkdocs.yml @@ -0,0 +1,29 @@ +name: ci +on: + push: + branches: + - master + - main +permissions: + contents: write +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + - run: pip install mkdocs-material + - run: mkdocs gh-deploy --force diff --git a/template/config.mako b/template/config.mako deleted file mode 100644 index 106f0bf3..00000000 --- a/template/config.mako +++ /dev/null @@ -1,54 +0,0 @@ -<%! - # Template configuration. Copy over in your template directory - # (used with `--template-dir`) and adapt as necessary. - # Note, defaults are loaded from this distribution file, so your - # config.mako only needs to contain values you want overridden. - # You can also run pdoc with `--config KEY=VALUE` to override - # individual values. - html_lang = 'en' - show_inherited_members = False - extract_module_toc_into_sidebar = True - list_class_variables_in_index = False - sort_identifiers = False - show_type_annotations = True - # Show collapsed source code block next to each item. - # Disabling this can improve rendering speed of large modules. - show_source_code = False - # If set, format links to objects in online source code repository - # according to this template. Supported keywords for interpolation - # are: commit, path, start_line, end_line. - #git_link_template = 'https://github.com/USER/PROJECT/blob/{commit}/{path}#L{start_line}-L{end_line}' - #git_link_template = 'https://gitlab.com/USER/PROJECT/blob/{commit}/{path}#L{start_line}-L{end_line}' - #git_link_template = 'https://bitbucket.org/USER/PROJECT/src/{commit}/{path}#lines-{start_line}:{end_line}' - #git_link_template = 'https://CGIT_HOSTNAME/PROJECT/tree/{path}?id={commit}#n{start-line}' - git_link_template = None - # A prefix to use for every HTML hyperlink in the generated documentation. - # No prefix results in all links being relative. - link_prefix = '' - # Enable syntax highlighting for code/source blocks by including Highlight.js - syntax_highlighting = True - # Set the style keyword such as 'atom-one-light' or 'github-gist' - # Options: https://github.com/highlightjs/highlight.js/tree/master/src/styles - # Demo: https://highlightjs.org/static/demo/ - hljs_style = 'github' - # If set, insert Google Analytics tracking code. Value is GA - # tracking id (UA-XXXXXX-Y). - google_analytics = '' - # If set, insert Google Custom Search search bar widget above the sidebar index. - # The whitespace-separated tokens represent arbitrary extra queries (at least one - # must match) passed to regular Google search. Example: - #google_search_query = 'inurl:github.com/USER/PROJECT site:PROJECT.github.io site:PROJECT.website' - google_search_query = '' - # Enable offline search using Lunr.js. For explanation of 'fuzziness' parameter, which is - # added to every query word, see: https://lunrjs.com/guides/searching.html#fuzzy-matches - # If 'index_docstrings' is False, a shorter index is built, indexing only - # the full object reference names. - #lunr_search = {'fuzziness': 1, 'index_docstrings': True} - lunr_search = None - # If set, render LaTeX math syntax within \(...\) (inline equations), - # or within \[...\] or $$...$$ or `.. math::` (block equations) - # as nicely-formatted math formulas using MathJax. - # Note: in Python docstrings, either all backslashes need to be escaped (\\) - # or you need to use raw r-strings. - latex_math = True -%> \ No newline at end of file diff --git a/update_docs.sh b/update_docs.sh deleted file mode 100644 index a4f452b9..00000000 --- a/update_docs.sh +++ /dev/null @@ -1,5 +0,0 @@ -rm -rf docs/ -pdoc --html --output-dir docs --template-dir template --force laplace -python examples/regression_example.py -mv docs/laplace/* docs/ -rm -rf docs/laplace/ From ade5f234be394a953146c15f823683596d89854f Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Fri, 13 Sep 2024 19:32:54 -0400 Subject: [PATCH 06/10] Move dev's guide to a separate post --- docs/devs_guide.md | 113 +++++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 111 -------------------------------------------- mkdocs.yml | 14 +++--- 3 files changed, 120 insertions(+), 118 deletions(-) create mode 100644 docs/devs_guide.md diff --git a/docs/devs_guide.md b/docs/devs_guide.md new file mode 100644 index 00000000..86d11ec6 --- /dev/null +++ b/docs/devs_guide.md @@ -0,0 +1,113 @@ +### Setup dev environment + +For development purposes, e.g. if you would like to make contributions, follow +the following steps: + +**With `uv`** + +1. Install [`uv`](https://github.com/astral-sh/uv), e.g. `pip install --upgrade uv` +2. Then clone this repository and install the development dependencies: + +```bash +git clone git@github.com:aleximmer/Laplace.git +uv sync --all-extras +``` + +3. `laplace-torch` is now available in editable mode, e.g. you can run: + +```bash +uv run python examples/regression_example.py + +# Or, equivalently: +source .venv/bin/activate +python examples/regression_example.py +``` + +**With `pip`** + +```bash +git clone git@github.com:aleximmer/Laplace.git + +# Recommended to create a virtualenv before the following step +pip install -e ".[dev]" + +# Run as usual, e.g. +python examples/regression_examples.py +``` + +## Contributing + +Pull requests are very welcome. Please follow these guidelines: + +1. Follow the [development setup](#setup-dev-environment). +2. Use [ruff](https://github.com/astral-sh/ruff) as autoformatter. Please refer to the following [makefile](https://github.com/aleximmer/Laplace/blob/main/makefile) and run it via `make ruff`. Please note that the order of `ruff check --fix` and `ruff format` is important! +3. Also use [ruff](https://github.com/astral-sh/ruff) as linter. Please manually fix all linting errors/warnings before opening a pull request. +4. Fully document your changes in the form of Python docstrings, typehinting, and (if applicable) code/markdown examples in the `./examples` subdirectory. + 1. See `docs/api_reference/*.md` on how to include a newly added class in the docs. +5. Provide as many test cases as possible. Make sure all test cases pass. + +Issues, bug reports, and ideas are also very welcome! + +## Documentation + +The documentation is available [here](https://aleximmer.github.io/Laplace) or can be generated and/or viewed locally: + +**With `uv`** + +```bash +# assuming the repository was cloned +uv sync --all-extras +# create docs and write to html +uv run bash update_docs.sh +# .. or serve the docs directly +uv run pdoc --http 0.0.0.0:8080 laplace --template-dir template +``` + +**With `pip`** + +```bash +# assuming the repository was cloned +pip install -e ".[dev]" +# create docs and write to html +bash update_docs.sh +# .. or serve the docs directly +pdoc --http 0.0.0.0:8080 laplace --template-dir template +``` + +## Publishing the `laplace-torch` package to PyPi + +With `uv`, this is done via: . + +If you want to make your life much easier, you can use `pdm`: + +```bash +pip install --upgrade pdm +pdm publish +``` + +## Structure + +The laplace package consists of two main components: + +1. The subclasses of [`laplace.BaseLaplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/baselaplace.py) that implement different sparsity structures: different subsets of weights (`'all'`, `'subnetwork'` and `'last_layer'`) and different structures of the Hessian approximation (`'full'`, `'kron'`, `'lowrank'`, `'diag'` and `'gp'`). This results in _ten_ currently available options: `laplace.FullLaplace`, `laplace.KronLaplace`, `laplace.DiagLaplace`, `laplace.FunctionalLaplace` the corresponding last-layer variations `laplace.FullLLLaplace`, `laplace.KronLLLaplace`, `laplace.DiagLLLaplace` and `laplace.FunctionalLLLaplace` (which are all subclasses of [`laplace.LLLaplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/lllaplace.py)), [`laplace.SubnetLaplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/subnetlaplace.py) (which only supports `'full'` and `'diag'` Hessian approximations) and `laplace.LowRankLaplace` (which only supports inference over `'all'` weights). All of these can be conveniently accessed via the [`laplace.Laplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/laplace.py) function. +2. The backends in [`laplace.curvature`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/) which provide access to Hessian approximations of + the corresponding sparsity structures, for example, the diagonal GGN. + +Additionally, the package provides utilities for +decomposing a neural network into feature extractor and last layer for `LLLaplace` subclasses ([`laplace.utils.feature_extractor`](https://github.com/AlexImmer/Laplace/blob/main/laplace/utils/feature_extractor.py)) +and +effectively dealing with Kronecker factors ([`laplace.utils.matrix`](https://github.com/AlexImmer/Laplace/blob/main/laplace/utils/matrix.py)). + +Finally, the package implements several options to select/specify a subnetwork for `SubnetLaplace` (as subclasses of [`laplace.utils.subnetmask.SubnetMask`](https://github.com/AlexImmer/Laplace/blob/main/laplace/utils/subnetmask.py)). +Automatic subnetwork selection strategies include: uniformly at random (`laplace.utils.subnetmask.RandomSubnetMask`), by largest parameter magnitudes (`LargestMagnitudeSubnetMask`), and by largest marginal parameter variances (`LargestVarianceDiagLaplaceSubnetMask` and `LargestVarianceSWAGSubnetMask`). +In addition to that, subnetworks can also be specified manually, by listing the names of either the model parameters (`ParamNameSubnetMask`) or modules (`ModuleNameSubnetMask`) to perform Laplace inference over. + +## Extendability + +To extend the laplace package, new `BaseLaplace` subclasses can be designed, for example, +Laplace with a block-diagonal Hessian structure. +One can also implement custom subnetwork selection strategies as new subclasses of `SubnetMask`. + +Alternatively, extending or integrating backends (subclasses of [`curvature.curvature`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/curvature.py)) allows to provide different Hessian +approximations to the Laplace approximations. +For example, currently the [`curvature.CurvlinopsInterface`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/curvlinops.py) based on [Curvlinops](https://github.com/f-dangel/curvlinops) and the native `torch.func` (previously known as `functorch`), [`curvature.BackPackInterface`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/backpack.py) based on [BackPACK](https://github.com/f-dangel/backpack/) and [`curvature.AsdlInterface`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/asdl.py) based on [ASDL](https://github.com/kazukiosawa/asdfghjkl) are available. diff --git a/docs/index.md b/docs/index.md index c8ca64f3..7b2feb3b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -48,48 +48,6 @@ Additionally, if you want to use the `asdfghjkl` backend, please install it via: pip install git+https://git@github.com/wiseodd/asdl@asdfghjkl ``` -### Setup dev environment - -For development purposes, e.g. if you would like to make contributions, follow -the following steps: - -**With `uv`** - -1. Install [`uv`](https://github.com/astral-sh/uv), e.g. `pip install --upgrade uv` -2. Then clone this repository and install the development dependencies: - -```bash -git clone git@github.com:aleximmer/Laplace.git -uv sync --all-extras -``` - -3. `laplace-torch` is now available in editable mode, e.g. you can run: - -```bash -uv run python examples/regression_example.py - -# Or, equivalently: -source .venv/bin/activate -python examples/regression_example.py -``` - -**With `pip`** - -```bash -git clone git@github.com:aleximmer/Laplace.git - -# Recommended to create a virtualenv before the following step -pip install -e ".[dev]" - -# Run as usual, e.g. -python examples/regression_examples.py -``` - -!!! note - - See [contributing guideline](#contributing). - We're looking forward to your contributions! - ## Quickstart ### Simple usage @@ -384,33 +342,6 @@ torch.load(..., map_location="cpu") output tensor of shape `(batch_size, ..., n_classes)`, so in the case of image outputs, you need to rearrange from NCHW to NHWC. -## Structure - -The laplace package consists of two main components: - -1. The subclasses of [`laplace.BaseLaplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/baselaplace.py) that implement different sparsity structures: different subsets of weights (`'all'`, `'subnetwork'` and `'last_layer'`) and different structures of the Hessian approximation (`'full'`, `'kron'`, `'lowrank'`, `'diag'` and `'gp'`). This results in _ten_ currently available options: `laplace.FullLaplace`, `laplace.KronLaplace`, `laplace.DiagLaplace`, `laplace.FunctionalLaplace` the corresponding last-layer variations `laplace.FullLLLaplace`, `laplace.KronLLLaplace`, `laplace.DiagLLLaplace` and `laplace.FunctionalLLLaplace` (which are all subclasses of [`laplace.LLLaplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/lllaplace.py)), [`laplace.SubnetLaplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/subnetlaplace.py) (which only supports `'full'` and `'diag'` Hessian approximations) and `laplace.LowRankLaplace` (which only supports inference over `'all'` weights). All of these can be conveniently accessed via the [`laplace.Laplace`](https://github.com/AlexImmer/Laplace/blob/main/laplace/laplace.py) function. -2. The backends in [`laplace.curvature`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/) which provide access to Hessian approximations of - the corresponding sparsity structures, for example, the diagonal GGN. - -Additionally, the package provides utilities for -decomposing a neural network into feature extractor and last layer for `LLLaplace` subclasses ([`laplace.utils.feature_extractor`](https://github.com/AlexImmer/Laplace/blob/main/laplace/utils/feature_extractor.py)) -and -effectively dealing with Kronecker factors ([`laplace.utils.matrix`](https://github.com/AlexImmer/Laplace/blob/main/laplace/utils/matrix.py)). - -Finally, the package implements several options to select/specify a subnetwork for `SubnetLaplace` (as subclasses of [`laplace.utils.subnetmask.SubnetMask`](https://github.com/AlexImmer/Laplace/blob/main/laplace/utils/subnetmask.py)). -Automatic subnetwork selection strategies include: uniformly at random (`laplace.utils.subnetmask.RandomSubnetMask`), by largest parameter magnitudes (`LargestMagnitudeSubnetMask`), and by largest marginal parameter variances (`LargestVarianceDiagLaplaceSubnetMask` and `LargestVarianceSWAGSubnetMask`). -In addition to that, subnetworks can also be specified manually, by listing the names of either the model parameters (`ParamNameSubnetMask`) or modules (`ModuleNameSubnetMask`) to perform Laplace inference over. - -## Extendability - -To extend the laplace package, new `BaseLaplace` subclasses can be designed, for example, -Laplace with a block-diagonal Hessian structure. -One can also implement custom subnetwork selection strategies as new subclasses of `SubnetMask`. - -Alternatively, extending or integrating backends (subclasses of [`curvature.curvature`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/curvature.py)) allows to provide different Hessian -approximations to the Laplace approximations. -For example, currently the [`curvature.CurvlinopsInterface`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/curvlinops.py) based on [Curvlinops](https://github.com/f-dangel/curvlinops) and the native `torch.func` (previously known as `functorch`), [`curvature.BackPackInterface`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/backpack.py) based on [BackPACK](https://github.com/f-dangel/backpack/) and [`curvature.AsdlInterface`](https://github.com/AlexImmer/Laplace/blob/main/laplace/curvature/asdl.py) based on [ASDL](https://github.com/kazukiosawa/asdfghjkl) are available. - ## When to use which backend !!! tip @@ -439,48 +370,6 @@ For example, currently the [`curvature.CurvlinopsInterface`](https://github.com/ The `BackPack` backends are limited to models expressed as `nn.Sequential`. Also, they're not compatible with normalization layers. -## Documentation - -The documentation is available [here](https://aleximmer.github.io/Laplace) or can be generated and/or viewed locally: - -**With `uv`** - -```bash -# assuming the repository was cloned -uv sync --all-extras -# create docs and write to html -uv run bash update_docs.sh -# .. or serve the docs directly -uv run pdoc --http 0.0.0.0:8080 laplace --template-dir template -``` - -**With `pip`** - -```bash -# assuming the repository was cloned -pip install -e ".[dev]" -# create docs and write to html -bash update_docs.sh -# .. or serve the docs directly -pdoc --http 0.0.0.0:8080 laplace --template-dir template -``` - -## Contributing - -Pull requests are very welcome. Please follow these guidelines: - -1. Follow the [development setup](#setup-dev-environment). -2. Use [ruff](https://github.com/astral-sh/ruff) as autoformatter. Please refer to the following [makefile](https://github.com/aleximmer/Laplace/blob/main/makefile) and run it via `make ruff`. Please note that the order of `ruff check --fix` and `ruff format` is important! -3. Also use [ruff](https://github.com/astral-sh/ruff) as linter. Please manually fix all linting errors/warnings before opening a pull request. -4. Fully document your changes in the form of Python docstrings, typehinting, and (if applicable) code/markdown examples in the `./examples` subdirectory. -5. Provide as many test cases as possible. Make sure all test cases pass. - -Issues, bug reports, and ideas are also very welcome! - -## Useful links - -- Publishing package with `uv`: - ## References This package relies on various improvements to the Laplace approximation for neural networks, which was originally due to MacKay [1]. Please consider citing the respective papers if you use any of their proposed methods via our laplace library. diff --git a/mkdocs.yml b/mkdocs.yml index 96fe3201..89afc3a4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: Laplace Approximations in PyTorch +site_name: laplace-torch theme: name: material # logo: assets/laplace_logo.png @@ -6,7 +6,6 @@ theme: - navigation.instant - navigation.instant.prefetch - navigation.instant.progress - - navigation.sections - navigation.indexes - navigation.tracking - content.code.annotate @@ -84,11 +83,12 @@ extra_javascript: - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js nav: - Introduction: index.md - - Regression Example: regression_example.md - - Calibration Example: calibration_example.md - - GP Inference Example: calibration_gp_example.md - - Huggingface LLMs Example: huggingface_example.md - - Reward Modeling Example: reward_modeling_example.md + - Developer's Guide: devs_guide.md + - "Example: Regression": regression_example.md + - "Example: Calibration": calibration_example.md + - "Example: GP Inference": calibration_gp_example.md + - "Example: Huggingface LLMs": huggingface_example.md + - "Example: Reward Modeling": reward_modeling_example.md - API Reference: - Laplace Frontend: api_reference/laplace.md - Base Laplace: api_reference/baselaplace.md From 2067e72ad8e6d696fec2fe9da3eff3f1f1e0e6d0 Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Sat, 14 Sep 2024 08:09:19 -0400 Subject: [PATCH 07/10] Add enums to the docs --- docs/api_reference/enums.md | 1 + laplace/utils/enums.py | 57 +++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 59 insertions(+) create mode 100644 docs/api_reference/enums.md diff --git a/docs/api_reference/enums.md b/docs/api_reference/enums.md new file mode 100644 index 00000000..c2da2bb7 --- /dev/null +++ b/docs/api_reference/enums.md @@ -0,0 +1 @@ +::: laplace.utils.enums diff --git a/laplace/utils/enums.py b/laplace/utils/enums.py index 6ee492a0..549f885c 100644 --- a/laplace/utils/enums.py +++ b/laplace/utils/enums.py @@ -2,44 +2,101 @@ class SubsetOfWeights(str, Enum): + """Valid options for `subset_of_weights`.""" + ALL = "all" + """All-layer, all-parameter Laplace.""" + LAST_LAYER = "last_layer" + """Last-layer Laplace.""" + SUBNETWORK = "subnetwork" + """Subnetwork Laplace.""" class HessianStructure(str, Enum): + """Valid options for `hessian_structure`.""" + FULL = "full" + """Full Hessian (generally very expensive).""" + KRON = "kron" + """Kronecker-factored Hessian (preferrable).""" + DIAG = "diag" + """Diagonal Hessian.""" + LOWRANK = "lowrank" + """Low-rank Hessian.""" + GP = "gp" + """Functional Laplace.""" class Likelihood(str, Enum): + """Valid options for `likelihood`.""" + REGRESSION = "regression" + """Homoskedastic regression, assuming `loss_fn = nn.MSELoss()`.""" + CLASSIFICATION = "classification" + """Classification, assuming `loss_fn = nn.CrossEntropyLoss()`.""" + REWARD_MODELING = "reward_modeling" + """Bradley-Terry likelihood, for preference learning / reward modeling.""" class PredType(str, Enum): + """Valid options for `pred_type`.""" + GLM = "glm" + """Linearized, closed-form predictive.""" + NN = "nn" + """Monte-Carlo predictive on the NN's weights.""" + GP = "gp" + """Gaussian-process predictive, done by inverting the kernel matrix.""" class LinkApprox(str, Enum): + """Valid options for `link_approx`. + Only works with `likelihood = Likelihood.CLASSIFICATION`. + """ + MC = "mc" + """Monte-Carlo approximation in the function space on top of the GLM predictive.""" + PROBIT = "probit" + """Closed-form multiclass probit approximation.""" + BRIDGE = "bridge" + """Closed-form Laplace Bridge approximation.""" + BRIDGE_NORM = "bridge_norm" + """Closed-form Laplace Bridge approximation with normalization factor. + Preferable to `BRIDGE`.""" class TuningMethod(str, Enum): + """Valid options for the `method` parameter in `optimize_prior_precision`.""" + MARGLIK = "marglik" + """Marginal-likelihood loss via SGD. Does not require validation data.""" + GRIDSEARCH = "gridsearch" + """Grid search. Requires validation data.""" class PriorStructure(str, Enum): + """Valid options for the `prior_structure` in `optimize_prior_precision`.""" + SCALAR = "scalar" + """Scalar prior precision \\( \\tau I, \\tau \in \\mathbf{R} \\).""" + DIAG = "diag" + """Scalar prior precision \\( \\tau \\in \\mathbb{R}^p \\).""" + LAYERWISE = "layerwise" + """Layerwise prior precision, i.e. a single scalar prior precision for each block + (corresponding to each the NN's layer) of the diagonal prior-precision matrix..""" diff --git a/mkdocs.yml b/mkdocs.yml index 89afc3a4..2e2c779e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -91,6 +91,7 @@ nav: - "Example: Reward Modeling": reward_modeling_example.md - API Reference: - Laplace Frontend: api_reference/laplace.md + - Laplace Options: api_reference/enums.md - Base Laplace: api_reference/baselaplace.md - Parametric Laplace: api_reference/parametriclaplace.md - Functional Laplace: api_reference/functionallaplace.md From e94cc5247d82bcb2f699bf0736ab6a62006dd652 Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Sat, 14 Sep 2024 09:24:12 -0400 Subject: [PATCH 08/10] Prettify API reference --- docs/api_reference/utils.md | 29 +++++++++ docs/assets/laplace_logo_inv.png | Bin 0 -> 15255 bytes docs/css/mkdocstrings.css | 42 +++++++++++++ mkdocs.yml | 105 +++++++++++++++++++++---------- 4 files changed, 142 insertions(+), 34 deletions(-) create mode 100644 docs/assets/laplace_logo_inv.png create mode 100644 docs/css/mkdocstrings.css diff --git a/docs/api_reference/utils.md b/docs/api_reference/utils.md index 10724bf0..0aea4c22 100644 --- a/docs/api_reference/utils.md +++ b/docs/api_reference/utils.md @@ -1 +1,30 @@ ::: laplace.utils + options: + members: + - "get_nll" + - "validate" + - "parameters_per_layer" + - "invsqrt_precision" + - "kron" + - "diagonal_add_scalar" + - "symeig" + - "block_diag" + - "normal_samples" + - "SoDSampler" + - "_is_batchnorm" + - "_is_valid_scalar" + - "expand_prior_precision" + - "fix_prior_prec_structure" + - "FeatureExtractor" + - "Kron" + - "KronDecomposed" + - "fit_diagonal_swag_var" + - "SubnetMask" + - "RandomSubnetMask" + - "LargestMagnitudeSubnetMask" + - "LargestVarianceDiagLaplaceSubnetMask" + - "LargestVarianceSWAGSubnetMask" + - "ParamNameSubnetMask" + - "ModuleNameSubnetMask" + - "LastLayerSubnetMask" + - "RunningNLLMetric" diff --git a/docs/assets/laplace_logo_inv.png b/docs/assets/laplace_logo_inv.png new file mode 100644 index 0000000000000000000000000000000000000000..f19295f06a36866095537be0b680fc90ef4bc253 GIT binary patch literal 15255 zcmdVBgG)M~wl1hq#lt?#7gS7OW z%kTI75BK^!$Ftpg?X}$UH4;|N z@xnXf95yK>K3RtuFJ&U<2aa0)#QUj>E(#go;~#kWV?=y?jyH3kR-LP5&u2SbRwf!2 zheG+6e0?SS?-hzbOkw`tD+|m-!GA9(0vYj;7m}fL5m%8H@$~ zB)EZGwQzj^_pnAbYETFE=ka z&L<-BfQ-|CV6yDfr!?`uvq2;>X#YO^-5yZgVOd4k z2gW`JHw+uy6OKG$7YZvY6MWAWXbS|m;TXDJDIyHUoR*f%Y4?SgioO>wUJPwc)M}Cu zARpeU5WbO1mEJ2CT^WvL0GmyO1R32Gg?Y(u+-b^-F2bvA_!2k`y|BH>DJetpv_3Ce zk)K8#2TT;vt+FI5eHgQIIo@3gpO`yjMZQZD-5!uTI5;RhG&Dp=Tz7Kf`R&^`=_?rU znaqEQg7G9S!qlZ@WrJSZO=Qj@Rb$HdwBpL@TJW*tZ~y5cy1DA=YD^|a85vw$T*Bq9 z|B!Qv!g{1BXlU|BjQtFq8m#)>4mh6wa?GRqcenYzY-+_5C^Hd%nsWGp=HTM;l8v1` z;WP40=fL42I!^e-roA5`S{fQe&J90)sMLEMIYee7JIqT9t4&oa(&85tRaSi~8j<>0 zhzjpyRn;#N4aqVEZt{q$>O;fBvW_k1ZteE3UcD+TE{+mE-KZjwMTY~Z?m!q0gzLHr z^YTjHzkh#!u{-I1pFD&Zz|=4{Ru#|WGP`n5SXfEoxLbAQ}jm0_zF7vgsX1hOBI!uxbQP!|XZhlhue%u@aI>62;g>gML{WToWR zLKJ~qmRfv=Syq)*pN5;8Kxuh-Qrq0;NoCaM1phBN#l4p1G;v|FW?o+D6ftt$<{^LH zVTS}B936!!?NB1ABk7$zIK+Rj*+U9s2 zZOKmN*VPHi4!}D3-?e`w%xP?V^Xu2Il^}d^0_EA)>FEid1FEy(%}Iw^lPmHv)sSO> z;SdbMA67czsRXaMxVVJd_|;TZ-G?hl)kGVN?Cg{~nGM31u@?~&dzhD(rx=TgLgk_> zP7;2$<{vyiJzD))q!~DYhmSvm@B1fc5d&S4L|Iw6-?qGo$z~!Z23OeU#O2x5E3hKe z^LbwK5NWBtw_-?X!E$JT-+{o}KVHr(pyNd`nDeF;78Ztc8q`Uqa2b;}&77Ty8rC_{ zM<7V|jVpwC8#$qdb~FcTG*r{sQf#V7^aRSJ&#=aB&1K zEOL;X!|p5h>({Tt7X%0Xmv8Z;KZl1q-h6g_@#0p%RTPyXWBKzd3$@wVf%bNGG9^cD z@p*Z-wY9aGJSGW<2yLyc34Atb@|&BDcK^}`t6@8f+`YQ8wM9CmXbORU4FjWcmn0Y6 zv+1qFbR)$O3uTwHg+Ti1cdiy?X$z>D7jzLxJaRb+G~(V0U9r@}lfN8lhlUDNpITZn zp(p%DKm(frWzpsIaPysQZ9_wPenG+N)>K1a6mf+=WRwxzu@hbN^xL;IUnZ2rGvjKi zs}*S2gJs(HJ;l;rzn*W>F4eX*fh7A;F|xn7$jFzboS1n%m>@qQ5d)P|@$uuUe@R+a zRi(MBL%%;UF>&)yPi%7G=ki{Nou~KDrYR+x2TCp2EU9*Q_)-4QDq|d(G=mvi za^xK@u05k-qw*9AoX~fhBErIfADx9Wbo{T<2hSyoqR78&w_Lt`VG+2EZ)R&-ytpI1 zGGbO4++?)r=JoYZE{DfDy{6`Fci(@~m_r8LSN%l=?%u^>a){>nH8T^Ja4*$OKtLeW zpcpRe?!9|qQ(fx^*FGBR=_Q^~(FLnrQ=&nMzSAy#5)~B{*gbK0==Qag1#?I$y_eLb zw5+_m3c0PPf>g1e5Ank8?>_e@kWORqK>fM*v9>mf(g&JXlBhQ3aYJSq!cUu; zNIlnYsdf4q1WHffPyc~1AK$GyRUaQ86R)h}X9UuiZLr_+Dh*SdY#wkE&1`D)cHQZ5 ztXy4PXFW6Cz9n^S@N;)h(nSJLV~ zeheKHB#=%Oyp2z{Kut**nc8UY=xA<4@G|7lqetq)EZ&D3y0gNLHa3Z(zL@kq+;E(^ zy@La3qidvuof_WSf#=7HiqGWcJr=xJ4t^DOKmPowSlqdfX&MSwNMU7fe`n8KlrX8b zvXUoXjfFApMymo_vXTHVFHxNX+#&R%os?dPs2*#6C0{&P$QLCV$xiDUy|kg#=+s7! zCL_xlz1wj&r0Mb92%Hg*cyOc3@uC90Oc7N|3R}kTU`xWHl-B4eTR(n+b`E6=yyr>(*V>=7( z6$KI8YeBP(wFwD?-3!6oq&Ki-XTNu^t-UWq$GJ#Y`SeMsdm)TKnp|xcHP{JZ^|88| zgvVpgv&mWm|4DJT1vaCF8TKwDF1q2VMi)M8lS21Zh3m{rObkqp(M>w8Y_4wzI&Jjz z^=I|U%F0M5FHEj4`2}?8X26SxV^J$Rn9T zDiv6m8jo#eUzajd(&F+J=mKhR{?`v5J|q;m^Ji|yz5j^h+mGGd-FJt&(M^T_k+D(? zt-8=35HK?{-~M;;|>xqbn1eGlW1O$Xn6c4Y( za|u#)@l=g*D4sz(v9q=gKYBsbr2D_MhKD^KmJLd+*CohL$A~)OWfD!atkjV%J4foh z7MmtztxCZHHI`^}-*VEKSSUd&|KnYh*~gNS=$hJEd9m%@zQI9_Z~x(% zn!Gp>g8*7}O~AG3N``z(Z6;f8aveWEzxE>qNGlp`PZj4ohx7ccQVbH(KgV?tA^73*?>-vwBj4aNDfyj3E zx0R`>)b4YI>T5w#yX4siv+r%88`xyFL6EX7 zIeqX4tzA^BO{Gri_oYbA^}o2!pFjU_W}d+Ud!E*}9JSgc68&>Ik!50h{7NmUrWi8^ z4L;Mn^c30IZf}Fl)2E8MJ2QX=h6waeFDh16SFwbcYbU4=oB`plZ*JbmP`KS1bln{;OLjA7 zWc~h+$A!KI*j#Qaz^z0SLaK#~}q5Mm;=);XuXqKomBOF>38Cq%x z`87D$igWEM1Q1VC%d@Rp4ULUGUMi)frP*<&l{%wmXWso6&^i@k^z$?-H#avwugfO6 z&`icvlI08*dH2c7%Lf@ryjCHY>$j;%G=4CkqpQn$a&-N+4_&Y`;DP9sj@n;|UY$|pYL$je@b^6={5b+K`c#P0I~hJ7QK`F zxo}2eW{?&%y2G>M=hH{B%H4r&x0JmxlPeYE+iIT83#wE!hN_*5mXD-Y*Wq5ful8(n z?>L)HtP*N3NwFH?tc;gC`ti@7FIRjXGHH$;8K~T;%hnAKFXjj(+*pcM zoJX(8%76-gGJKV(Rg0@c)^d4qTCp9`HW%)B5l`@p5*>*0k{CvoMXQ}alwNOoi7X$W z8L9H2$w{gM9=w#w@Fnt{uU+HguUF17FfiJhx1X{0Q*V*_if%`})PD4+a(dbbn#}DJ zQfEfMF}jZ*lR-!)olVm95NMtT3TJR`jX#N&l~77xrD-;F`_ndHR@v^NOrM>W%9-cA zS4kcv>i4iz8)MicN(Sn+#=N)=o&~_W-wxPeic4O8f>>e-ets>5T%-xjKW}@{D>7`X ztr59W^SqkQnE;Fx6CE8co}v@}#8ogje&lD9j;%+1I6D&80`pcG zj9ckH9wd_I&?noc2miF@H%+R9rBC~iV9So`56&sL?2h;Me{MA5x=Bs_@GIu>+$W`l zg$35jT5=ki*fp^jjM#-yXrqIcE~Be1k&zl+$8%=G~V zi4>FoK40tLj}UyEr_pUEa8qAL#{@bpF6qOuc!k~GV`R784Sp5PaJwdNx3)+Ix0rp&*T)M$?s;^2JWmVH?*`eM!&((vp1=czExfJP*MA)TUb-_+qJ!iu31Y#kU>}%z710jfbPG58j~bmE08cGK^H8= z^jk9FcY8^BImYn^@ zFQ*HAO-*7x%9w1@yZih5?Nr-e&|)k9CVp`WWnZN~DBX&T#1`P@Cir`_>n;kXj~o8= zi?>w!d$e56ik{8R;J`pawa0VmoYC%$HTv9sh9p2L!G0g*UYj#7kzL1(^WULm)JUgj zmI#&3qn>nlG5R($&7sah$F3VQMEZ*s+l?WcnjRDGir>F~r!w=%-v_icI8EHlkVDla zONEYL=%iBo7Zu^{Jw3`jl{WmZ-=T5pE-|*2etoEYr$Ods@7k<8)O$;aeykV?4O6pD z@|!d?D#u>$;Wqh=5Q{Nz{0urSB((_R4(WX1?{x49wq=ws+{w@X?38*~KyR)dmzz|a zLalh1aKbS!(V{0oM6jSI4k}(<-`s0M@8@i7=W`KOXiHAaG1hu2vbJ?X*4^X>4_=ws zCO%YFuE@&Ds1?!r0iW> zxjrn7{dhiLp{`J0?nz>Tc(9MSaepa&)!3W(A7l}X12gr<*k7@4c#3k4rYQ*cLC;XK6 zWA9W~Ec~H8bSfP)TiyE1Y;4OA)|L>jI4>Rgz<*Hy_lq@u=w>(T;m419rIrv*b~ZL_ z^EGK-wN<5a0;O|0B6dx_3rK*D0P+RnQY^wVMxU#6sgMxR{AWJlY;JyQF-1@#PN`*1 z+$dFqFcinQqcM%!Wv37b-{yVQ+Z=T6sqE4D2z=C1sWmb>Mc zUa0k|RlP{7al7f~h;vD7ObmJkB%fSQ7cV(ZNlwnKTnAl-$Mv+a_b6I0q-idAKh)R1 z{z*9NWPb_76~T_UQtmP_zRbJ;({LG@;%wn)q(#y#ox7FXDVhzRx0sDacZgJ zVJ2_>0c(DyvsBV3**`}n&dxkfw}-y;MT16|JKb7RRD>n6yW-ZMD=QnaID#82Kf2;J z+YrL&)l*uNc?y zLY4B&?CtFd9dG6w-VA=4^*84DB0S14pgpC~39e)K(xoaZOX^QtfDS1sse20rJ4xib z4|rw|UQvPkbp5b9IxqSQ0@Qdwy}I@B1g=%8RyIVg?eBP0grfV}+M2s#MyIAcr+R4d zCwHxN_WW^$ykxxjy;IfYg`i6#g>P)b4H|UkwJM2ft#wbR(_-5`2!2; z-4!_5n>)##;g-{ys;a7AFN(IFr*2W=gof8z3C5b=x^=6hsVM_~ zz6v;*fT(CsXIIw~v7*t;;5q|N^${5l506=&*O{4QcD`k5Ebc9m!M2@KfOZ?sM@Mz^ zVx44_7-4aYaa=>VwI8&@cXoDCC9a;|s;E?vg>WwU_))+tbKSzN!JmNX=hvHml6k`| zTRDvt33-Zj;@)2bk4O|eWKm2gz7`a4=gu9T%sysetOMgH0IdN64$^Y(pDxkf7+60r zT|R|!B-2=$jwi=<$Z-^cx5{Z85oewD_N^tzN7>uTR3+D$`O5*a^Cfk+Q&Lj?BVdQN&TgP+*j$EakAQ}3sa+2!|1T#61lAan-N$fe?Z&UU)AG&OJ8_%URI z{=&w>@@{g{;EO$lc{T7y8JW}5Q{;u;<&hul@Or@F!04#Li>=8l^R8Gz!h|m-@4Dh> zuWvmDWR=2cm<>?S^JKrMaKz^Sle9ZXmiPDhLDw2eVpcU9mly^GOwHNZ`Rjt&j4^@8 z)vH%s_I~#UQ}-=B8_4M>c`}@@tVhUttciczew4els z*Yiq5s{)&Rqhi(H5|;Y@5U8x3g9GOpp&|%sLwDy7Ja4m-5tS?D@h-Y~*Xeh1?}X|% zQE`EU*<)784c{!$wr#DiwzIQiY|P{cRwHVY=)RW{!{c0gzil}9!9D1`719FeG%DYK z+um|;tQsTla^^`7k6iav=;`gPeEpg-e`G!Fi;0Oq`kmjMKUe+L zp1YDaVSEq!i}jfALCyf$xKD?^f));l5DinYgzTwA!!~!JOxZns%9@p_@6A|sP8!Tz zy6DG1SX{Ok-fu6?V$@9fcrJ30hU3~Dehp^1=J{q{mG2!lC9V%Fz?}&1~@pW=)Di46U z=l+WNm7MRQ2ApIdvZ4DYnzJUKaGln8Ti z_4E{8X~G_PyE*Iq9ntC*Le)6sY$}=B4-W;t#Ym=lU^{AQYj;xNh0TVwpPba&3Db|- z3H;u1H#^~m;pP(%O%pTk3wt;@+%!Kc_68-O|LiSHT#*wwyjISyYYv}}A`Yg=`x|ou z8@;%+^hsM=8$B%!&;*T41?Bg{HkMUmqfjh=(uS%OB#C()a@fmLYc9<_<3|phpfcL< zm%T;MY1McG?Nv$&LK@*D9hlU?yLX$fTIgkI#mKcmL6eCQrLlbWOk?-2rakbwey{$i zDe4csbI)!eVc&&o%0yl!;aY-&kAEvGNX-Z|XuXNSL89R>laeO!B}?e?Un4)k5G{Ta z_~6yP=Tt8i8H9JrE9gm6c!9h~I;JTlPNH;1}a0>ihoUdxwv0=Sz%O&-&MD>w}LvAp->%wD=w z`{oDo_T{~uj}^kBe6Aq#?~c-ovc2xL{Oux4{I-X>ea>n2BYW^ulXhWef7;?|=3UgRR*;JJO zjM}3|vbByg`SbJhxlM9F;F=D8=_`teqa?|@`4m&9cVi?uIXHstx*qE`mzF++ftctw zxSn{**a(0^A*(LJ-F?KZIV>>5mw^~s`Bkbf6$eu_B#47(G>f!;j*ly$qoW6OLCmkV zhFoWjAVv{TBz4Qc97#Mw(3yd?%UDN;F0764ZyJwWdV2c)Y0Kp)ea->Ozulh^FF2&U zw{Loyrs-_=yOe$PPn)~v@VUs+@88$#0Y#^L(D@HF-@odTtH!bd3LJEWhu>iQ+`SHX zU=Rk0uoM$dbQG4%?wn+*uY;Z4pJ`zs?}=QPhB{M`eV`WBlVrh%0fVwD&qBwhTNO-8 zwAIz=kG)V?(J<~xwWKN8bRz)SzB&Vh@v+baK}IKnZ(7hbf!}VERap#9Ot2q+y3e9{ z8y}UQU|A)!c=gx%L8!O>?};CqZdL{u_?U6opk>_nq)z~P;_e>g!EXW-#@|0}_(8r# z;753D>~nX8Cr0jm9E}*Zr=7WZkkW$(LBGHKhg$50v0(%#P`xR)73T6^=X1G^s*U_4qiNR$*M-Q|K5kXhK5U1TPEyx zI?44xMuYi+sTl`|y-Z0F@XNVQOjPE8LJ~7Yf}h=})r=Da5oZ5z!V$w4bT%y=9r6!T zpadb-j6ZzFUyPylsI!ple>c8J+}+z_m5|7M_Usu87grqeKW64IY;5eW%X>&ojO)7J z(9lrb(D2~+_!zmd064%{z@WGRJQ+rSD@m&235y5+eMPINn8V})TS0lth3BkQ?7|8R zVR6PBAlz93V9Ez3fOw#mQUI5xz4v)4;mCs)9J-|1_)l=>5GX3`s($XmET~;-|&7;w2_FMDsk6C37L=ZEx z+`D(rdhRns=;rpeCQSJ=gr0|%8GIu8`QL3LODn7HO*djfb$Fjz!dLXqBr+uCYTDOO z?wT4K%NiTg)zi^=*W3H>WUaWIh=>b3JBc$xAw>trd&^SKHN3@XhphSOCt_k_ui20E z_tSZ75ac%?O3WK|(zLQNP8blF3FCu60^%&pA9!f5N~ap|FmHwPaC1xEbDYLGYiwyz zz1-!xls5J|ckOpW>=Qw>Xa3HU8rGtj*Et1{1{ z5p%`6f0Yw!61K*$#W%y!(o%zx9rcV+2^cZrsFIz7U8g z%<~dJF$KdBo~NCuDSe>{*FaQ4eVq6S%L-5I0t++7x*_5oalxq7!2;nI=k$inEvq=-^XwH`e}w)%?1_V)G=wCli+12#4L9c0GA8xt@)_i#ea zr2KjFCr;a@o2HbM)T~ufH_6M25O%rxni^SVl{X5C|GaJCw93BU=s=2sBu55hXt9oj zL77mTbGs0SkbiqXLLE*P_ztw#9?dSd#jHUih5>Kr)Y0MQL^mEG9D~9_1J{N>QleP` zZZQCYs=1>(%J? z?o^loPs4QIT@HX?BAD^D1YVQ!-5O+2l^@Ob!hkV z^wHzTL(G}piNWBjLHS>*LRW8sB!CuNR;Hnj-|q zLQF^gZwgCG^BP`kUy~bPfHlEs!p9E_E1H{;ff14)3e-wEuxNTPPl4`X&USkcsHVip zKpX;iA^GHT2Ks?T7FJikgvOeO$81hcPX126)&a|*VAy6A?1(P&?O1w7M*leOY9)Qd zgZor?^2+Q=Flbsm7%^6rmVr&sh-hK&>rP@W%FPXqr52F`*j`;(DFst-*p*edn=WfV zS=m@wBcATuB#8UIl#Nu4-9^Rz1-;T&VBezsP}tUopmDHCmHVCBo<=GUNZ$p$dvPCzYAuh_98d^?0C;*Yf`AF?F!+__I3gEZ8i>$-1K{ngMhJe zrf24yf};m@OSS8r7p@UD=_Iq0GCDsE?zPOpmpJ3j_gPzEbOVT5CRj1g$6lhX%^$+Q6!d z3m7Mbea{2|xCQ=rBXobcFEd4$gHE$Vn_bLvU%^QPe%fCt5B#7sL(-v8=RF~+l92mA z^$@PB>2Tb3A9i%8BZ_P_FU1J^CpEZga#~xZ+1S{!_m=yH;DU4U$Smm09)!SsxC-rg z1?+z6+sn^BDP*ZJ(t{gOkn-;2^?)R-j78PujMz9C5$|YP z+?X9xvQmf$1)aUeLd-q1+K7t_Y6Wsgd&HzT)GUx50~bt01@+E?6Iuu|kq1Bk&d*>p zyj(FzSi;OMg*P(L2_n9II7YHAPU1rF3n94DEI_P2-<-L)16}HYB>aF);O`<|Xvy)rT~QK# z-5~jmyqp~3T*QiosFwz9O3T1N84Pg%&CRCmK{Z-wI-0)jg#&Z)M1SCI7{cG90ptGq zMGEJXEBW-u(C}fV_pV~U`Q5vBp$rVaYfgjBwWyt7WoM59{PVDc0R4j{q~LrCm|j_# z-oDcF4ZcwS4@IYk!0I{L3;HI&Aui`9t^=9sZVegJ9?R6z%S=p6D>WlVIWX2iB1|Yl zg~5}j;NGjg&ro;9M@Qd%PvN>HbX7hy^J2fiv_TBgmUfi2%_7PbGqYusgU$9mpl20=GV>7)T^v~*fFe$OS z1^!Id+?E!J-yTlVpt7y4uRr*EMuL*G)X>ztf3!6fCFHT!_SmpFCnsoYy6NVK^M5F= z)HiQ(VO@LTVIjDz*-*uwVo?zycx~n4{Is1s2PguRLInQg0!Ehf1P`%cgIf&1qB{T> z2p^XgYq6am2~bfhVLmt8ZVT|-Z z_hf6fQM*#5+C$~O2kb>{DD`ZpFze+lvFqfPRcU304gc_E18$e|&!M6Fcws?rMLncp z7U?imAF_yzq8J1lEU092K6r`-*g8;>QjC3d*SM`MEP{cDE>LE%b8!_kHj2&*#`^9= z8Pg>w@ItILpKgf=yKRu)AR`-sA5#IoB{kthB@fz8^~aBL4z=lkB!OWk^eyKAey$)k z`%eN~9nHtTZ-Q@jx#TX%*@_v6s;^vTngPFCA}6a zE3-fNfD3``KyU_tpmlh?g-b*-vOpyxdt<|H9N>FKMuy}Y2n8)IEg7Vsf??*%kqD*s zy@~RmAF8;w~%bs zpt=I8fuv>sVK^%qR9hD*~6T)Ef6-Vt2P9 zv_CM-D}q^+{#Gy)ko{l!j*7gIt3U{Rec!@*BEnaC()a>p$?#j9slX&k+~usknPT_x zebU0itFV0h$KJ5)}E1}ucyuMf~zre;KM$LXTAw;aZ2L%NIW+ToI$arcq5k6=mL5r< zy61C(tN}iDxhHW+tSQ98=!})0l@&lE>Q3N%vl4646k0`w$ctD2C#z79Dnl_PrN_p{ zvw?9UEQui-EZ(Mi2!A;|Jp5`|g|59|%TEcKAR)DgTQri{8yo$7zK2!uBE|?6D)2ig z2q1dki~pgeg;pniseG>vNT86=Weee}%S)uyOH#01ULrKZ_+rVpbLx~z(cp?Y0A?^W zIj866GksxH<$(I&yZpj8%we)FAasx}99a#B6|f-ycsXFboMj6T21C%D-cdzQY{u&F z*S~)s2^BbMz8Zz<2B0x($djoE?SM|YRvn;JlJ=fbmRkSi`OM`2B5^`G0=Z`qN%#Af z)3Q&4+^!blyG!=PKCQafTqrYa|_ra+JPse1x15 zx@4VP6xh}Zp?M&FZ5b|`@dAOi@lz5$lPSRT7kK zpl^lC#N;^F0AV6C6!dzbH=-WAX3y^N@!bbZ^N%Qs7HAi*wp^^YWJl5y;^5*A?C-mb zyIvEDhKCTM$hZb$Zr*G4{i1*Y6FqAX?~DNQS6ioGpu%G^&lZ^}W+;gi4Fa8nGXM`q zRP-?yzZX;B1e-h99KWs~L{h#gFDlCVM{X-YvOoayn>TMhZ}#dWk%MHDlaqtcD(LGU zHlu9#DObNGvm;ys_&f4GP?~*Ez;9Pi>F|SCpl)t%t`Owu_n;m@eSnDQ*K|PqGc6EA z9wPwVXsXeJ*p=}=1qN6EG)V?maz%6qQ-ZZPw){2TuT)S;pq9l)%bB+`K7g=7AT^Sz z2*eENB7-Y@_AC+jz*8HWNN9BFsh&T+-DsFn0kKnMd`BoY2SjN-z;v0HXUpoU9N=`y z1H2r6k1#Vc6W^xFszwh1`hu~yK$&!zc)ijQeiuatk>a)jfVjG@?yFbWBFkqJZ#+D_ z4{h#EMu?}41UqDLN6JG7d6EEUH5(?Z&_3+IlC=0@oesK%a={S#`P-*3NYK=51Fy>% zX!|A;OG{6{@=Q!*?BOA_Ajrw`3PyB=aYHZ;-8n;{vcUWUqxsv%%UZ(W33X_*Eg>Hv zona8l&&QXvbH5IiOBD1)5{jhD(zhE(ZUTg2cCA*yK}zrr?PWq_RMb;Q4J9R|Z1K?4 z8=G`{X&D)L(8J)_FmHn{q||^k3_yUu0PQ0v3BV$&o11A^QW8K|M#j~kz=5p$GBcCw zq-J?}`KhaGGPF}f`QonqJ1e>X!2s|y8ys{12kD?&gFtu`8`5o&otQ{e@4Bu%TM0@R zjD(5AMow5j)P+ZGEId3YuCutOrln;8p&6f?#5oN!0;0Ci6?b(H-$$GljNqVn26Q}v zK_c{Wo=(8nuQM`k!-G`7<#9;VTR@GQO}#+39vd5zy4*IdUNT~jCvp#YsoK|x>t=2XKA)iv(KSm1^rZ6RBhkwN{e zVQO*59jvh5-I5Q~{X=xUhv31kqwN`0XZ~64H2peI1S}+lM|z@CfNTvFWcsq5e56&7 z0FBP^xWw;om|r{{rBiCp`98EIia$Ucmw}ejyOLuCEeOJ%X9HrxD;nY|Lt`Ye6?p|2YkXa>~AsI+XiCt)w!ax;yiGZO3aX@6$ zJsx0sD)$^F6-=}wf9P=@{WDRVPMn%9#D#?_3Dg1WMUoY2LNc9zl%MC_p2N zPJ-6X&I!#YYwSh$(NW>>^dIai${$B6'); + -webkit-mask-image: url('data:image/svg+xml,'); + content: " "; + + display: inline-block; + vertical-align: middle; + position: relative; + + height: 1em; + width: 1em; + background-color: currentColor; +} + +a.external:hover::after, +a.autorefs-external:hover::after { + background-color: var(--md-accent-fg-color); +} + +/* Avoid breaking parameters name, etc. in table cells. */ +td code { + word-break: normal !important; +} + +[data-md-color-scheme="default"] { + --doc-symbol-parameter-fg-color: #d3a81b; + --doc-symbol-parameter-bg-color: #d3a81b1a; +} + +[data-md-color-scheme="slate"] { + --doc-symbol-parameter-fg-color: #dfbe50; + --doc-symbol-parameter-bg-color: #dfbe501a; +} diff --git a/mkdocs.yml b/mkdocs.yml index 2e2c779e..c43d655a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,19 +1,52 @@ site_name: laplace-torch + +repo_url: https://github.com/aleximmer/laplace +repo_name: laplace +site_author: wiseodd +site_url: https://aleximmer.github.io/Laplace/ +site_dir: site/laplace + +nav: + - Introduction: index.md + - Developer's Guide: devs_guide.md + - "Example: Regression": regression_example.md + - "Example: Calibration": calibration_example.md + - "Example: GP Inference": calibration_gp_example.md + - "Example: Huggingface LLMs": huggingface_example.md + - "Example: Reward Modeling": reward_modeling_example.md + - API Reference: + - Laplace Frontend: api_reference/laplace.md + - Laplace Options: api_reference/enums.md + - Base Laplace: api_reference/baselaplace.md + - Parametric Laplace: api_reference/parametriclaplace.md + - Functional Laplace: api_reference/functionallaplace.md + - Last-Layer Laplace: api_reference/lllaplace.md + - Subnet Laplace: api_reference/subnetlaplace.md + - Curvatures: api_reference/curvatures.md + - Marglik Training Utils: api_reference/marglik_training.md + - Utilities: api_reference/utils.md + theme: name: material - # logo: assets/laplace_logo.png + logo: assets/laplace_logo_inv.png + features: - - navigation.instant - - navigation.instant.prefetch - - navigation.instant.progress - - navigation.indexes - - navigation.tracking + - announce.dismiss + - content.action.edit + - content.action.view - content.code.annotate - - toc.follow + - content.code.copy + - content.tooltips - navigation.footer + - navigation.indexes + - navigation.sections + - navigation.tabs + - navigation.tabs.sticky - navigation.top - - content.code.copy - - content.tabs.link + - search.highlight + - search.suggest + - toc.follow + palette: # Palette toggle for automatic mode - media: "(prefers-color-scheme)" @@ -37,11 +70,7 @@ theme: toggle: icon: material/brightness-4 name: Switch to system preference -repo_url: https://github.com/aleximmer/laplace -repo_name: laplace -site_author: wiseodd -site_url: https://aleximmer.github.io/Laplace/ -site_dir: site/laplace + markdown_extensions: - admonition - pymdownx.details @@ -62,6 +91,7 @@ markdown_extensions: alternate_style: true - pymdownx.arithmatex: generic: true + plugins: - search - mkdocstrings: @@ -69,37 +99,44 @@ plugins: default_handler: python handlers: python: + import: + - https://docs.python.org/3/objects.inv options: docstring_style: numpy - show_root_heading: false + docstring_section_style: list + docstring_options: + ignore_init_summary: true + filters: [] + heading_level: 1 + members: true + members_order: source + merge_init_into_class: true + inherited_members: true + parameter_headings: true + separate_signature: true + show_if_no_docstring: false + show_labels: false + show_root_heading: true + show_signature_annotations: true + show_symbol_type_heading: true + show_symbol_type_toc: true + signature_crossrefs: true + summary: true + extra: social: - icon: fontawesome/brands/github link: https://github.com/aleximmer/laplace - icon: fontawesome/brands/python link: https://pypi.org/project/laplace-torch/ + +extra_css: + - css/mkdocstrings.css + extra_javascript: - javascripts/mathjax.js - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js -nav: - - Introduction: index.md - - Developer's Guide: devs_guide.md - - "Example: Regression": regression_example.md - - "Example: Calibration": calibration_example.md - - "Example: GP Inference": calibration_gp_example.md - - "Example: Huggingface LLMs": huggingface_example.md - - "Example: Reward Modeling": reward_modeling_example.md - - API Reference: - - Laplace Frontend: api_reference/laplace.md - - Laplace Options: api_reference/enums.md - - Base Laplace: api_reference/baselaplace.md - - Parametric Laplace: api_reference/parametriclaplace.md - - Functional Laplace: api_reference/functionallaplace.md - - Last-Layer Laplace: api_reference/lllaplace.md - - Subnet Laplace: api_reference/subnetlaplace.md - - Curvatures: api_reference/curvatures.md - - Marglik Training Utils: api_reference/marglik_training.md - - Utilities: api_reference/utils.md + validation: omitted_files: warn absolute_links: warn From 3ac556bdea54779d8241fc8323f390f84e3dbb45 Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Sat, 14 Sep 2024 11:00:03 -0400 Subject: [PATCH 09/10] Fix nav --- docs/assets/laplace_logo.png | Bin 19074 -> 13550 bytes mkdocs.yml | 20 ++++++++------------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/assets/laplace_logo.png b/docs/assets/laplace_logo.png index 82c4ed0642b7bd34c277db6d5dcb9228b132375d..4b483aa191f70442cf2880ac6e4e03dd900d8575 100644 GIT binary patch literal 13550 zcmcI~1ys~u*yfK=5EYbEIz~{sB!(1`l5UWOp?ii#KoAfRP!IuWa7YJ4LWY_1Xh|Rlx~!}uuMI)?9^mx> zF+TWvhbY1n{CB}!$p{HSq~Ff|;U#gA(m~LxV4)LuceKf6@u5-#U1p9 zAW0ctcS~z01d7=TVdvl~#ky9DWo34-kzzHtt;VnBeivcypycm~(D7H-wf1+i7Pn!Q zk!F_kg@XZH5GYG#Ul(UrB-~eu_0PC)@cQgyK33*GT~JO^tar{H$ZV*l$$Zz%6TvLb zE5KvT&o9U1*l!A3EAt|J%>q%hUPKJZ-G`5Y7k}gewXO`U?CzsJp!z$_;7n_P-hW->?6} z1HeJ3sr~zm|8y=cF8@9PiMrP zo(|xh<=Os8u?n2+g**JNC&ChikO%*ySkDfThhLmWKuA|W2rhgZE+odoFAV4B|EsH- zn~j65-~ZHASXY1_F2oN9hxR{q1x93JiL(4ZcDAvG+q!wWSb{}4xLDdD_}pFXSegIb zD)?PDXE#sqU@$wOKeJF%gDbltQI@XO2xWOGRz#}Xw zir^6!7Z%_Vu(7g22#H&X2nq@R^Lu$WYp=7@`{(yI|Chf9+X#Hk()qtU&Y1_E9Xnjf z0f}<+^!wXEbPyhYzjAh9{&UiBOY6T53u)<%uwniCLx=zJ`u`f##~uOB|KG#@Ur+F_ zZb&y)l`l5jmg80IeN~%ZGk16+>NU)5I*MUaPiJ-q?f*dW|F=IOnENRsKm#`egFQQI8+m^03I}OF|r|a7!@2mVOw#FfdNcSV%@*4 z(dv1YTE+02rWq|b{Okc1f~n+DgfeORoK%;Vz;vH+QO8^m*-^;TLf<8TAbW~Ova-&G zWnBw8fAydAtoE!O|*)xxOI10zugf-)20b9>ax%@=zgQ;>8P|)r3o| zJ^K3k^rVr!0%z+^8+INaAE$21*T|IAI@=j&pbv*4kF6UW9R*XxT|1iwA8l+jqR&PU z=Utn0Hn8kAOkwByj7Lmk?q36oiiqeJ>3&mSxqj(9n&52jq_VmUIFkq~a3dqM2WnuD zPCJpb#Fp0drREkse}47sur<-C zsn??_>~b_^PmL5)Q&OIZG2VBR* zGr76n*^&3Ovhp^1<{YG%Xu*F|;9c_k=Kc#(3gh~<-8PNF9(!G!FBrYSwgLg{VROE| z{*hUk0bJWYd}G67|ALpZv+AD?*52Ow2d}H&y$h|cWBBS=Sz21!+Mb`J(&$>n#UKcU zz{Tj}pFd~mA+xgNaZWLdmEj7zYulHhb*ii`4lN=MdQvXXtNG|i@)WW#SeoM& zjKAs#L0I+-HQ+GqwiUesa$Jk3nudnEop_qO$-~Ea%jCw@CZufr5cK)?Vh>xV6w!js z!QO^7*i-5aU&$kDX7Z@FyI{B5^HFe&ZXpEyBENLLQTx_Ct#ZRlyZH>^I&Z7Ai&WLs zfBRVfXp#~Wvw!#u7`C-wYC_9vLlGWIk`&?|2mD7660#9Y?dj2~9|KNUr@{K%-s3R@ zISNZj@puG8t9UXr+g1n=g?#(CYA>v^R`L1sceoPMX4l3VWufbLB>Uchk4h>zai#O!pf@BMx$g`3UqmalxaO@0hH!W;z9!sD9 z@Zke&t_A`~lcsDit!<95!lBy4 zmip6U(r^9koYXN^8H!9pBWmLvrFmSVFVY5dOUpc{wE#ZwmSD#VHjv2IQ($LUgwG>%aTAKUKBJzQup#^zy zqt(@)cD}wT!%dErAH?J!NY6sS?RTS2H?AB0k3Ccd3I#o47n)BW88}$N;pOLF! ze3T?N?_Yp2ny1|985lMy9MIbJvT7Fm;Td^7g+tO?S2ki}W2wKNgR)7zYfaxQp%93B zxJ6kW;?V4}nz*<)m;$V4g}%17mWPxKlC@MX*qoe{j$z#&0Ep$hMh5&z(C6FFYeTj>a@2K ziR1_USYLlTB)!b$=;`_H%x-Vq0w{i?U!FDRYy{O$lTn{JK*=MGFKP?(~!{>5L8|^=L>J;;K%C7-FFsDnpm@Q4AbSLikfS}gLp%M zf*L`c1dwGx-9gh zLW+&RV<)A7kv+^&NO(QE;;JpFUV!xTOGSRnb>)ZNQw9*uSHtFMAzKbk^iZQ1DJ4S7BN@ zy7K|flrBgl($?12(^GqWTDh{qVpwhQ5+E%o6Ud#hcFc@3X|_N+0aj2@FuIj6Jm;z0 zA2cn?QLfrf%#2IrZy%X?9}!P(HF49;#bu*lm3aKCh)6_)@9){DNxY)7F*0#DhtvxZ zxrH`@!#^Apf<(|OYinE#?N>RkLmz|_?m0+ypbhQfP_p~mrG1zeEMzJg&Vc(6E zk}fPPoQwaC!SI~XBM-)Sj;yY>c3$8koZ?}F-*G6wqf(h(s^PwbtEs8=#1?ie#=jv! zrn1(JNESHpQg zd@zybC|{Frqy0dwu)12@+qcn5tFxlrb`uat)JO@QP%clZs>-r$rSY&pOot(cs5bL$ zy~VSk#8Zx!<5+|$J4Nj>r6OmNHsDKFzKY+vpLo+}oL5XOE=RN3wW@Z^HFYAan1qIj zk^!YerLX@M19z7XQ#<(1@jjN}s7 zIQ_<+STx0=*M>u>Fh!T~$u)#Yw{D4o{x4Jtsb;P!`PqEW5l+?(%Wb5wcW%g)=Ji_rY5AuxR)KJAa2; zPOeh{*R-<4-#(rS>vu}ctqbo13jhdXw z3SlR@K8*nENH2I>5Z)Z9q9>LmkfDBmt;?uADc>1KjvIK=bu6!TTT)U|Mg|L(Hzv$e zsCiR9Ppi(X%;E@&wGl}x$%s>KsomG)@TOziEZOmu6wKn?2^S977H~-Klv8Mc*qD3H+TO!fuXUnHhi6Sqm9x)f+=*q@95~#GBc^UsPp^rZ|CLZ_10@>XjBO@=Jkk4Ntuoc zy9%(ev0;vgq1dJ}A5I-^Y906vtw(;#+Un{L{*SuTX}6M8Fj&)+)YPfREGaL^qaJT% zRi608qGo1$d$sjUm(yEVIu-0;&Dih5Q5pHg7mX<8! zQL12#8X8?49m)olPQ6SF*;L;+UEBKeFX$EYdE}4&kkq?7-|A# zv@!Y;;^McEnf}sQWZ`n;O(Bzu<_A$7hs01vBQL8C0itQL|7k_(?HhTy&r*;NriMsK zNnZ)S5V%6T-uHU)ig`J%fd2L+GQXYW>&HLkS+zDV^9Hk{T*{E8{wnIN9w=1*QwI^k zJ5TDYg(4IxTB=D{ls!RC6c}jI>B0MBI|L`(UHV!vAoKpoTSL7m4Gzq{en4R0r;%y} z-MqmNp=+0p*gt9)eKRjs)#(td3e}*=8*s$M5hTEtoag2r_w>tE9!e1Fg-QZ1D+8}8?K2wy%_S9@Gwy@^^&PmhqK82^jiQ(8o`gr`~CafWTz^TBmXs&7(F?`i~0Ha*`_Zr-Qrf&_NJ_f z-sSViw2luSKQ_PVt$6zuD$5d zozDIVuybyqUT_8>*ke-)75=v?D=zzZ*J2gebAHJ9g~^yypA?mEi}{$KDoM%6nwpxN z;uPO565i7S)b~-=t2I6XE@THfKIA;^f6L6)2FZm#j~aftbM?O-ID`8ee- zAn?QLnGJK^PW@VbfH>fDh?19c7u}IpP(Kwt?bjb*Agle|UmbotJGtxe!*Aw+4DLcQ zAa!kRZ3F79?KXmy=h${aJ8sG@qTuI*COSLSJCggY5Dj$Wa-`jG>X>k(Y>XOWrmGT& z6qmUk45+TtYZd9HD2P{$n%>135n2^H;?yb<2nUMdHRKRIj-&INXmnrhkA_{Q&xq(lTfo>gF( z3j<6d;=Sy;Q?Ry5nH1Cw0SxMSR9}5L_ogEPflkU1N!Kr-lDIxl1{}t!9B2cbe!$q< ziqg47R1Kc>;cPfW>ggym)SGytZVYj^U#|*W{Rr!Pqn`jv6HbiYoGZkX)!PUTAeWZi z#eVt0gS7}11XJJaBd)i1+4%TWv8cTRGdK!ZvMMi3@CwIsY;t&)$oQejp|I#zu}KC% z$DtObI8}BbA)%RV|GPO?3UqdUet!_k{JM?1*}Ofnf;7tFUH{Vd@@p<%t` zwRL*@&we?kn3$O5-QsHMrh7L8nEZo-v7IypOYQ>`tmr;mettgTgJOYWN?!a% z3>?SLV19V|0`yEwG!hibqZ1PgCOQ;^^{VC1ux8Qq7FW3Vxsb+R0cY8K0T@|*f}29# zbB7+P+N4=s?H5YMLD;Omp$7w>21eF76sp!#7j3VkpKB`A<=pOPhbh0C7if78Hm_VO zs4$9*MqZ4*^jGt|iK*FB{a3UjLFoJVv2k%fy7wITui-!X(%#dica{SKpnA;X8JSQ3{t}M?cdh+^kfznwP_3IIovG|${o$Zg2iURKQT}n?$T+O%c z5sb-|%pmk0oOodnYC)Mn}0m5Zu!O1fMo`etCI$YioG9z%Am_nHGfekCP4@TqAFg5f zHpc+WfE)|EHyu-K<@yHXIEOnfJ8ZoZcNL9HOad1&d>jK@rL)Y9jStFB56ZGmj~14j zz)`GCV9iIJfU3#I#}|BnE2xxW$i6(}EYhj>8s&5;m`ZPFkqV>lRA~%iv0P3!2qQyZ zBL$OG(4YY>!Tsx7K_6~N@8pKc>|WsKu`yF|pV9MScjmM5O?XRes#!4lZLTFvuU^p~ zZ43>luqQIcD;D(m)s2s|Mj|SP{aa}W-czV|-MlgUvMuY`hzbB;8Vh@myticrK!73K z%`v#2B(v~H=el7#0O1*FMFReY6d)EJED&BMBddCsav|2tnSQ1`r-In5frQj%+rQ4s zY+Z!#e2%^FD-^sVki{ydyXXF0gWEUEJ9qhl%`{^y`xmR<8?Ouv8R{f1fiU>(+hbIu z)R%BXrEy~r9iaaJ6ZA_4T&sxi@%Htw4`kn535=Y4Q_`4a4>Dt1apZt{dh%YUa>7%2 z){hrIDP1}rHn=ob_DHKj0rMQ{E_J+iy^!1rlR?8+V91QE!Pkk&i@4V>?!{;iFu{;~ z)~aH3Wqab&yr(!?hA5=>8}&E)i$D#%T=M)qc>ymG);w`c!ff-Gtbg4&@TL{-Ei)g= zldpHcrq(c0JSfuDhB-r24jXU7O?V9!lq{qNyw`5i2Xwfr*!N1Ah9{Ai#Sv-}WTJl)hlnn}9Ox!>SUPo|U z2g!A(qZK*OY`CdoLVHB0`|hi3C`hEKvghZO6vZUr^MhT{ORoOdAJ%#A%HytunKu*S zQc@U{iFuBJ>`WQejho4BQhvYXkLT9VP&7`X?hS79)50~4H)A)(Q=M~{ttxgvKf15>|9LKj8ZAY zdBZLx>F8CJ<0J|`9F{OtLJ7hJ%A+%vCgh8Y&E*SX($X0m5JH z`?y!C>^quoRM1~Wb0I5~w+y!Q!L{`u+y-{R+F5AW*V(eRImesYednm=IL5C5V z^Q;~!NXpA&>y+aH3fi#$^YbDN_XH&+(^b&42}S@^l1_x7?o2VCp~J(&WlTD!h~WyU zZl`1AI}W`;?j)7e4qLR$W#~QC&YoFxE+C%)CF%vBj?(@EI6iusurRu9&QZx}9jHxP zr)jTV0mC!oqJbWJ{HjZ8L+o6;99I7&)7+dlnd%Z%_IY643!A_Mp23WQgAfmljIPts z-IXDR0>jP3(__ba5#@$r1uW^(*%&`b##Cf$i5@pK4K=m1NLnw`RfzJdsu1Gin5!+9 zT{F3If>NQoB z|sXh?pn8J^r?8hKp6;E*SP&-B{V6>KZmdM~aGy=@?bq zo#7260bv9aN0v=Xc<1PMGWJv8{_nZAgqaR z{ejZ)k4cJw*NnNIVkQnSbfT^|$-lI;D2t?ZENr5_@+PSaJHOlv!?(wq)opx(!AJ$} zJv$LqPLMeq?ov=xOvL~O-M)Rev$F$26czDRczloj(G*#*@5)H!_&BhbMWVADu(K1a z&uy$`dwr?_Sjvc5pq*WTA-ALOtJT%j`uaNu^~q-bxL)*qLGvjPTY;<|9DKUJxsVln z;^$%seYv*@%0Btn0_`IAm#s#oric%5@$p<#FcV)u9xkt8DJf$WynxVO_}&;Z46LQ& zcNRXB;pE}*Tzv2u%t0`9=s|LFGENQX(@So$o9@L=l-lMz!Jf1va!k4VdU$v^IhE*g zqQUrLl7fO(<%XzJGNA&zY_FkUzNU?Qim`QzMSxB-E_79PEo_o}zW}P0j>|=bK-#flT)3o?Yxz(S)FLe_ zDk@Yz{{(Q&$Uz*Mr3ut^&*x`l%ORlc4BbjJ?Gyz*LH-M9exW&MpAYu;Q*t;7rVWXf z33r<4fg$!tYRQr;%MFP`Euv?HrZ+zJZk%# z9v>2ZT5kbrJ3aROR^|QcTW8ggpUtHLL=3TY(I4Lfq(_IHgX~l6gf;M8zkAE2)jj6Kt`F!5Y*c}guAFmdHDf9C1NClsm zri05T4!dCob7f5 zij-hl;H_xS+A)=;i%>@A!seW(c$APHTKoBUPTlK12UOn&qEO~pje87COa^dJ64F%2 z1EPv#z|NbZ;rks{?($7_{_i|gG~jvbm{Wtx})E1Dv?Db`+?(YyTKcc zH0^BciMTK_Xi>k2;E^dG+}UP&_LzBf8?@}WJWwq|_{U)Gd3 zbT9sNFTTW}Z2Y6g%qm#<>Q60r?gz@4Mj$p3oj*VBw~&!0>ef08us-m23r9PfS=sJN z)x-~bVNkj~s1{pmqar2@I^7L6pYH@gfhlOuQU@#{rpdPADxt^H56t22TA#J+9V3{s zSq&;`e*y^HCsMq0p2ghWwZ~+q>tu-4BLHUy(dQxAXT#2n^k?mp_sR(7LDlMbT?tB_ zIeBpDcyA`Y@dIl?CXih3v0k+eeF50q{;!fU5eXQi$q~x@d8e}M;r?9mE?5K#M=n+M zuB^<>!eSB_k+YD(Inf;YdhricRS0(71X%E{PjJ=n1{9(Kf^LR-q<0bNR;*j#UFXC%JMD2) zF^v%s5jLyz#DsuSzoI3BZmO3J5{1=k7j11cF|p-%bC=JL3b(&J~nZ(G6CQZYyy*&)h58VfShsq;7swK)LHhH zI1Fz3{(bS-KwqErxe`2s`e@Gc$5UDc22fPD`2hAEEGxdLqPF(Izy&;BNgyHY5qz{L z=9GQ1Mw1Zqt=5j7Upd^MMg&Q)op24OdvpWa40I4FaTNZ;Gkl{ z>_4E?5rc^Zb?V@gh2YPgbFC|*RTKV8n3IzrTMFd_kbmP)HHO^!oJrvDRM3KOxbXtH zx=X#i@X4~B%+lH#&6|a}-NUt?)d1>18Lmi|Gr4Umxo>RjA;1a=pVd$M^J(q!&jY2g z<>k-Y<~Tu`sx2-dVQp3b;HB)Wxw*TqpBx|F{6Lh^;#w8FGi;Ym^jrr!?O7eP*TD2) zc?IO@KY!BeJTWThJ3gK`JqB(FZa#4)ksMDBlo(9-&cp&HQ*R(9F0Ra;xF2}3x0fO< z(_7Fib21~-Ip^tV#hzL=3u4w57T1hB0+)Y|dK8%kc2?;X@-{S1w=|ff1C>tKzJFJMXGn{S<9_|h z#TJ6+j#jxP!m8`)GTP=+D3#3FfUMuAafuKg0N&If03SeCA8w{iyk-hKIob#84_OVQ zyW_#&$j+R*7~#bT<^o4y+Sm?T3a$ct8a;SMW?Gtrl+*wY1@>J< zuglcf*!aN%YntI6dtu(>r6u#z0}${_d+e1GfPAe=f&VbLsKyfomZJ;?mMVTZ_+8as!jnx1TFLxf}~_x3qsWkuK;1w(r*4H;M;M%C#F%g4B++t z{-29sN~be`yKf20959_x_icmV+rW|a3NzAlDveiu{CGK;4{jU0bV6T`{VsEmC*yy( zQdMS-H8V$b^8=}`#l{oFt+`}#Fi&;8E?}YH7`AL)Tb185EVB0X-Kz>dtkT+rvi+Pz z=92L;^LOS zI0601%*YThP({_hdi19O>s4;g0SVOo`}cvPZvE^A4mwfewX{8GZr62p40tEd%@*_G z7qy`(w~3(j$@(@vEcae0SRmks0Qj~pg7oEnGH(w}2mt|s+ouf$y()X)mq0oOE|T*H zF0s1-SPrDLy-R0j=hvzAtfQ8!^+Rlj?dJBHaC zXzi)wW~&Ng446vr22C)_OU^|TL1E#35V4AM!F9x06ywm!&R^uJyD_QM9Lt-6sH&>6 zE+2{y+G5)YJ0CVB12(ejVGJ!dA0N10Q2CCNI<0A@8jy)mH*5#?nd775<6|H&14I6h z-O4W8b~MW&u>Vl5#YkJ_YQSZfV~KgDv>z0k*fiy83yuQzE76@|k=B2+c)w9&{Ci3is!a zn&O$b7>t|Bhn)eH1ve?(x$ay60C|?NPT0G-t$|Cbq>&9af)`#!8Hn%v8r}x#Ik-PC zjF-U^`honeNdAs`vc93n7dXdjf)#pu1^8JAC1?rd`N@RXlE z2JD!Uq;lp$O}=xTN7btXZ*YEDC%7!viO+Qv6+*(ou!URX4Kq0iZ*OmqdjmrFV4wmx zKDeX;)Pu(JET4Thzy(PuFqK4A5bpc~0xpF50t(n=>Jk7pXK}F*i^Wz|B{SZuzf|B0 zAU+1LaqxR!y!)TP4TG&_puGTl@|k#;ypIA`CH>`<=y2cCNub}Awx(11BLK@MCW23n z#7|Lf!n9{nc1>8db`i*?{XL}o71$o;X(Y@7g_rrtN1ruD)9r90%14Y-Lt|qdM`4Ug zN@OVoKG{sSlG`Zn^@JVas!MBx_ z+oHjnqDRN#T@2T*WoO-}d%W4&8f|eW4eXLEb9bQvTaA>${KyLkqN#6he;TS58vCxM zhL)HR=()Y-b^#!chnnXNbrV@kN>f3vRn^q$PEQV16j=f=uM@Yln0YxWRw^3W8c@Cs6v|0SSegewzqJ7@zMuMHFFfj30(`~6r`sc zOs85!77)ZsTxZtx6t15Jm{pWKZ+nWn9=HRT?nal$HY+>I2c>t}R&4L?3f&M22nYb; zG;Z3coj60^e`nc4joZAxQ=Z6&0ZMXH5J(rLHYQI(vy~9Fy&0(4h}p+ z9Tz`(wcS?eFs;lxoEl`?;Km9Smz@^Iko$WOBCH-bOVWV}L)**-fAI67qFd+&JZRCE z@Q(ZH?XJpCpWdMQ*QwYs5ppbNss#irx5u|`+^XEyRU{)Vt>WzR@I`FB1;0G&Rdk*` z*|j`E2s7g@1~eQrNY$*&AdPf)NH3j^Be;w( z_Nf3J-zs~Y`VgnpsD#z9>FBHB4{mk_FwEEQP~REciB%}}xRyW7CvaPG_V+!@av8iq z*kG)%ejkUER~(iTC)?P#BedcFn#|DE6hXwW*y@e?W3;G>YvBclW#bEh9*djSJ6`W^ zzS0Qfpg<4&X zpv1Gt_Lz2a;8CqqBp4kr1SOo?o5Isd`GHU|&(SPX_Kj`Y5h!;}LJ09jT3PU_294by< z@6Na8i7UQ9EG|v)IAtbZL`>(V_Cnj)L!`rNX*>^?HBgMVrMmr8<#!F0i1+9$aDokk zT|#q|({VvS!1@aRjhOrdiwx|UjZ0q~`r zxvL4ar=6|6i-4yv?VmFQz}N6^v(ZxjImPvrFs-(t3bmwzvpF?4D;Fy}tq3|bHMNkl znT5a$DVcvC2mcbLwQ_ZJ6kucX@bF;uc*^SFY{|yK&(F`s&dJ8f$pX$`aq+TuHSuJz zccJ@xk^inEW$t3?Z0+c3?O;z0zplwk2RBz?T3Yyx{`c$ebGlkv{GU77yZn1C@Pcgc zpRjSTva|i~+Tf!?@b3yJJ6oHBJHxLp!XfnM%>UQ-{@q824Sw_g>tX)B>7Va{R~11Q zV*B5+iJ(94VA4TA5JQlc5?A*`+|BmdAz1Ujqk14Nh746jY=l(zI3t>C@O(qou^Zum zO?*&Pd(v+=lf-D30kdAQv){v9C{)z7zpJ$KHZ5$lRy);rBdw_;bt;X6^FT`F&3#(m zfa8qCY?qC`^ph<9jlQg7|F`>RSx=Aqo(>*+uV-EKZCo&a(k*+bE+q#2-{CC1RHuKU znN5I*i}T-6i6$2@8cGeJ{qNwXr>0>IPyE0d9f%t7?+D>DK>FVav#gel zoq!51!u{VBz?uL5MZnVif3hYpLI`fp)1dUHmKiP^1K_WLw3HOW(Rv4?8n5GZA#8a+ zqQ6@iqz%>8;cZRP=GKsha!MH={rqmRF_69fBl!tKi$sE;`;PW^ce{W49wb9cO;i5H zqDH=p)2s)lCgAS2eYnnUc0#|&oyD=u{olrm(cvU0>eKbT#E&)kHI$zZA%7?nLp&@c z#ijnof`TjD< z{A2QojM5XT$p0+R3JMwd%H6@tb5892`W>PMOB1&o_CXxZmDh?eo&f z#=%1C+n>$Q{5w-o44jGo+?g`$e5YnGLot&FxyxgFs;YoQ*c+zP;_XHmRv;4f*I~rQ zfmf9Zyv(MKX`gF!>3_?h(8rYPyP}7g*>Jepo$0aYH|f#F^WWJucnLU@WlwX1oJ>dE zm)e8cb1=ww`gW#knEtP=c#5CkVe@p*Kf5b#(z;ahlLRuKbHTH~(oyIp?5}S&QUlK= z8XY6f&Ju7d*_Hon(izr{_~y)YqD;qP{UMSW2X7KpcGW}VYp&lsx zSMH;o2`Aec%RwH4&#oM^b$0i6za0;XI!sg;QR&JGO8^ z*4Ci39zI_CG0dRJ{S9P}E?(%Rq31F(nj_jp$>)DB;)&KHIIbX-i;vrWYByPKu-=#c zG{5@|z8mn4d^ecde@j9wf@r<6Qu9KQD5%Bfj5I}wh|Q4var?>6^k`qMs1YP9I(y;2 z9V3MnMk)lqOswVbJ$F3*f{?EgLF8g2rP7G?bhp~qm?+Wc?& z+JlhnFb&isM)LlxN}ddO4_&$;FA>Zyp5Pi3q&(2@(E0`MUUGhCDTsa!lioiU@4*BQ z==@ZpcK?@}vaorb?exoXqc%#tYEeMI-> z#3#M}i{lNX?vdJ8T2C$dh-+LoN7@DcXWz6c!T{Dwx_MGr;)ywExxzQ?>gO%G;;Aas zbsDn_+;sn}Q{Ec@Yp)8Y%Z!;5nUr}lc^!Co?dMQ!%>g`w4beWPO<;f+`JE(t0YB>t zy-(ElNw5AnSZK_AJJf&zTDZx?J{PmDC-rhSjj=)Kf`>l<;?JS!Dn9@-BDTSFa+8V z;;Yf}8QS3geVB})eK`Y4}L#Fwu+ z=W&D>3)=6!`PnQXFV2Zaf^J)8tOlPSl$Jk^gg+{3vxN&i~ z-6Y6DoNaqn%R+x&d}qq#s$cZ?u|*?H@*M{(d@`PCGfm zK0gFBQU1NR6+ic2%~@${L1;v#{PG)FN{)Du%VHsAGUEj@6I2lNxKOYunH8eLx-Kp@ za(R1GSYAONkOsp&oSqS&9!K>KdEn%undnlj3K{pPtm0@w7EWy8-ztAs46l88J(S3> zvet6%Rcbh{MDWgB?b=~6&Hxr zR2IXF{qNz}REBu>P=c_0_=;Tj=B9lQzYz_{w*198PgE=ZaU}XDRFP*g4GG1ZRzuXm z-H(*+11~3l_c4Wt4D3?)qdG`cGu3tmhKN`pDV_aXZ6l)?a_Eg{=B z;eQ~!{f-H2v7}Z$851VYS@?J=-TtwRQ!7~6l0Z=0^2)6@Aq$wm)@5( z9wZ@H2;060#ullJCt)`Zgf>8gTW~>k6=Vgej*PZYA19rH;Z}*1xNcAMH=j)Ct#20P z#+8)Zz}M4Lip{9Cc}eG)W&#olR>&!mY#6~4%_+BQataCT38la34joOo6495ML6KM8 zEm21T?|qz73b(j0vNfSQJNKVxl`|)_x9RF}2~|C_J?W7e`ZT`WX4jEb5&}Gi z>IHU&7~fuB;4PI1%i9os`CiEj|XS40Ru!BjHVB7RM61T7TJG2~_5!bWZY zD`L6K@w?F^S>v7r8u#@Cht>;5P6|RS?4Ls4HuLRf>)ccBp*gNOO{&7pV~H^jht8f< z!&|vyuQMDQrB11jcW1$j=s|>>Ag4s2)QMi5Z6rm(!C^NQTVbHjN+eU%O1HV0+*TOe zEPkd0rl^(jxy`{EmO}~Vr#gOF8EK;SW$}+H-JMQb=8~jH&&Of-`_bECDHLgy>vP;( zo;;O}BR?$tU_Ako z@CbEvM71>ehL#z;3UPM{p;Uz%1|qIpE_ezxvnXLQv|nwvm)q@{^6K#@Mk+54+L3A{ zp*GLkv&DZ72<~&?Gd_QazP`PSg|@%wcV-~yN7!5J^jgbjcF6LNZ3LZzJnEYzfi}J! zGHJM4krMKRN%%@j=*sXw3{E@l^5E^xpe+j7=LbwUY+wAaDFn>o5t{gg3(d?p$1%Z% zgaH^V@7RnrJ>zlxrBdUuzkohi*eRWN31vqX%am!{t()x|`>57mZQi>x=zmw4ENSfW z9HlaJ^ynkiBRhQfp8kf%%Jk>MJg`@jtYc33AfI2~tzS$*&|);D(A0y3U^?A*1ov&|}^$^&&jr9ObFQg3wjXEj%6sqK<_HhBQF4=G{dX zIvWm6&UdXio3BoEpw)7L=Ki+a6$VWf9&B+4vJto=U8)a6zINpWz_(O|2!hUmha|XA z7^2bOww>Z&2eVf~y|;)y^AJM9Fx2Q`>2h&o-sC|c=*~sPW6NIn75#A>=0n61`Ii-A zj^c%|KY_nQSuCRgr)%_sVvS@N91mM-LBZi3Ow%BA=zC~o+J%1NhxGv{I9PEZQ&;{4uo@@iV1q`aY+?nKaLkVir@!H%8)%+k3PKf ztywSLrl0mScno~dHgce(fA2Ohz=kRJVj_Y1!dCg4u*{I9}Ht; z%3B*VGC;6mRLT(|@#bH0`wXhSA%U4k5FG}1@+5fo=!PyiWz~Rm%%~ngmV%SB7~C2h zX5nI^-I1mri#19$K?z8T3UkF?f2LGvkd?wm>?7D&*Y>cibK>#Fq_?l8Xs%mOHUUF?Nrq zs!Z+TZHwM^Z1B`uGOB9bV99JYrG*X0QF0GjzWt@sQuz_8TD` z14b=h3K5PW9+CLG0|uhIXP$up7i+0uxA^+n(tqP(mY5`(Q;~9R-7$YkfxPi1*AF#r z^7p4vV!LR@^>sdHdmWKFO5a?mi<$KshMJFlJdwucn+mA*{cJN{lGj;MNAV^0fgwmC zSOri_0z8S;>?dfMFaKbxG?cVUaC}=si|a=Oc8I!mle?W1%PwJ%Ev_TV{(DQ+46fHF z-pAL~Kc5YZg;RPLTLhbuq@u7?Fb>Tk+8(!F?FsH%_`xF$BJMdskiJh*wSGuHw&*Fy zU=)Ntf^;W0d#Gs|xna^MVE%a~Pk$Y<6kV`d__I1Rua%C6b{mEP#lr1Tzveq=`SFSV_y%0yA=bW z3u`HS95p}e1hKmnb4BJS!a`VD`qMZ%nRTk^u{*6xM7(PUe5pEi7Sp!ZdQ*uwI`;Z@ zp{lSNbSh82h-b$Ht}`hqDR~Bsb8#&zDJ*)=F5`rkgwN*Gstw5Cq#SjG*q-eQyo9Lq zD=mDU#aH#nw7oxDaPo9Q8i5hfTb*Vn0u)72+7brhKT)xX|cQD1iPtCFlaeS8VEC(cDHc`*(>d-OI2#s}1WQx#**nNL#bcJUpisS!h2SI^1Wya~ zTYVdgJ8NBu4{R{2TFAJp5-N;IK74uwwIlENRc_D}wcc@)kK6T&GJXNjhU&RutRZ|m z3mrRt9qb^*?N7L>(%D`3?6Gf{axUzF>=4>JSw}LL3$z2^+9m2p|cX|34+3$wH_WOS0QHyy1RSI{T+|V{49tgO-*kBm#cO$IN!wpOK(IMGiuUxBYutOQ| z40D=ma9RhMusZ=)xKr&rLuC!vNbQ5yLxc+rP6g~-wqF2oqIRx7kO^!@YfgrLgXN5v zYuO)*!GHIQ>>oh+L6bm26IV2Gce{VblgeuF*pl4+`lAKm(on18vAg}H*jFDXQ0aFm z8-FoPU`L6wc#PrAuUlGqYEz3XNGFPj71Yx*wlPV?Np}6aNVFDd`7<(Jk$ra@MRxti zwj%@2)mYvrXujxquxNxEWfs!hBm$OjI2J{en=NTFj%PfTsNtBpzRv`^-8?8CqGtLo z!yA^>JrBxcKg5(BtK9*YTfyxK!P(#rjGH_$n(VOO1Id5e!jRNHYJErSRnHqbh62X+ z(p&`DZ-CyLJ{cZq;C{{!t4((F@ayvTO^;RC;)`YMfM2|Q9d%!VQNnG?wW~cg9Qx$xmrV>WHQt|T*7Y`Zn5z8Z=PgP6rr{e|%#@#g4(tox%L}yI zy+gkJmgcNoZ#Wwt#0TAC{fhfIAFrb9bayuM=IX4Q9; z;7r2K`p{A%l?4t0g^92`9f5jC6E9Q+4{_v^_-}VQHVJYGSN(oE>+8DVTaz^~{JR-|)PB4BKl_aqf}#Npy6aExg;}=RM((zA7bio1Zyb4@w#q`0+%JS-G3= z6zI7q z@z;eI}Xk550Je+nmw)ws? zA_7sFJ3P$pf3{rTdGx|z6ObaiUuqbGb;^#%@w*TYOvGn75zy>T9E6 zRWIhN_&RvQZiFG>GfucW%}%ubKXM%E1_Sj3hJ3~{$MWV$DaOE^2jz`h?jFy~@ zdQTeqj-_yEzs752%*dtuejP&7=7M5Zs@a3Ne_iED2!n>Sm2CW>IA!B_lzHkY@e1mQiA$215g-$H*S>Id;}E8$wH^wWyDc|Sofs(V(Z$pa z<4)bI7wRBYPkQgUwXQR$Q~~?x-tX;=<&{Iibwzqd5hs~A91}{VEt+C`AMf4}m#1k6 za;@s_uDqSqN?+CNhrUfwh?w?a*UpOfaY`PUG6U%Y2gX=-GGT{+xBghw;`c8H80^{z zPgc9=C4FMLxb6Km1P)uPX%jRL>vnJYYE^nW-dYWDD=wm{CaniMB?y1$%unf^sr^~z z5-B!;s!spaq;S{QdcH!nDq&7Re{c01_gMz#<5yx&HkGI1*?HN|MsPCTb8#9qTxea2 z>=xO#Aylgd<+Lcs6I)*;=T$Lsx2}7i2q?10o9Ifk-O|q!(jOFU^1Xp6%WroISW#x} z8eZ|(nhpOH%xP6>X{eCDB32jO@~wTa^12-PvIi+%>plh7hlbf<+k3IO$5m$Ka1b*{BeE0g$dG!7zVoxF=&F<#%2db|O%Cn=@-&bUoC_Iu(!_T@XlQLN3e6qNX zLyc@`?O$3GN6H!)i9=K}W>Shp3afIxPOm6T_EuS69Xob%avtxtSj^=PW4+gS2YanL zvjV#eY7Fs-X%1(abmrFNs&qKc8N48e=_X8QkH>*Z^HETBXagXcQhd`poz(@3urBiz zy^-8zUL|R@#b6A?n+hA=tn=E7J|6!-8{z4&BTmY_PL==N?oLKD7|OThGy6fLx6bMQ z>u{rthi&Gw4R(y5Q0Uq&)|-2-*=d+<`SN7xp6z2_cGc%qqezT z2*>4G9v?#(ZGFnK|0F*E4nQ`_dCW8e%1Z~VKIfRDrYOGwMjuziD<^wekH`@3EP5T;-eb zArPmlyJfI9d(hwImk8tb{${|;jFi8}h<)P4>FWMMSfY4s)DsEVJFO(H!bqED2o_NeZ-7EHg!^1MbUw&K6p*?r6G z_ipzH-}w5H^aJw!0@;A(-B2_$z89Xzt?YJbMpvr6cS-?Xa)+kG?jjpgHZ9f7J8Q`n z?7wVInp9^_nkbu(zu*OMMOLVxZQ&b)qB+EuqjcN4zn&5NRS!aey6U-#g&C_oh#=bN@Rr4MuFrsQp|5gQ36_6>_M!(@flxv8Gu@ zrUDnC3g|CPg%in#*k{!bCtsxT^s%*QlLw*bJ*mi(yStXyXy{gpJ`?948jVBLjIR25 zNN$p!JU7Yt8Yg3q>;9xq#zLLa;xEgj&Lq4kK^P!9PMC1?B37ffZ%tPG-m=v!;%Dbp zss_;4?g#l;f9>Vlv-BTZMO9{w4M7uhl8t8%H;!XO4+;~tw^`#WYB1GXZX~be`iXg~ zP*PvHQXdnKK&)xJY4E|EbqGVV(^#mC&E)FZ_IV@W$f+F0wq(toq;}lZO(A6|mJ^kC zLE4C+uEj%N%V68k3&7kwiBFx;ebQ-0)FZ9sw9nLOTJy}y-97vfkG;i8Bm(YYLQ%KG?%pH!|51$AIU zOFQSNbWo8snzWo<{Ph}p@)n-stKjT?n=Gx?dZ%)awgo}bDQ87h?Z<0049tVv*p_<< zGX6J1JRq^Y#E*EeaxFnZ&oT8)W%{asusgc;)@o}MNp{K2cII73(&g{!qZ(&}d^bts zsO{->{b*EwCDQuhoRDuYXBaJP&GK=(TQr^2RJHxs#j7e&<|&s23JRMq6>nY6L(dOx z7o`2VTm#o`8=l~!8Y^Y0x_?4_X*I(>y+nHEBjz+iJ5RNa1UKOenxUw2U89xumyOAp zfW;!%t@nS>2gDiCb;y2|wy-|Xn~iqYb9d~jZh1IEc>3jYWCoxs(m@|IM5E@c)@J;N zwsQBWkAkUb+g)_qd9>61jZ=kO?|pGj`g%tnjvO&uJN1YE_C$48zY1 z>1aafiK5r<#d&}PBtm0S?>pXuShrxVXd<@Ly&pd<6|}hlKf=7Jr{BteVk2n_Npbw6 z)M)ZomsUvQOc{;3*j>D}1N?0B+bvB#@~bA5>ViJZl$2?03y8&ZwYhm+UMIkeEg9

dH|*EvAli98G+7+kWPutyjh*dkkoPDCM?~ zN;a>q%#sBtblvu(O>-8S=m$B^`ZIZvP zh6Y7vyW-=40@F0+(_GX7}+0ugiE@t|I+u z!j!akFCY=^4SjaGO}&yxd$kwqvX3VsulVC|!r868E;8C~x7&DJ*83u7VOx=6I#5DZ zeX9k}RZLTs*o^x`tV!cCbHU89c{+g9o7*Y@I*Q7Ks65f!bMkHsC-H2`H+0kyxsbq>Tdc(Bs^`WlRmqukuQ z;UOetFGpr&Z)Cn+&7RaJGwiZ~1!TQjcEF)5B-tqP!^x^M`m2#<9=QSQf-?BRUVb~|1ng<4mAa-4eJ~PK7X~hx4v5cL zO0~J2i=YyBlSi?}-5Uv67`OM8k+3J|5-J=JAi1*+o| zRCNiE>E2f@@Q%$n{!%@Ryof_nqY^RMVj#D)E(&NFLeQNs`ktlPFZi9!OPRu6R*=NZ zXq9Qd`PHDSE@M3P+3i)?4z~XdiI8XIbiG6FV)=(f895|d^z7Fb?~zJSIy|6X;7Hf%2~&kS%Tqh@kCR~5DTV|uS+kaCUx?Tt9Z7~` znZF1uxFa4iJ&$G&cW1GlZVPDJ9C;s2Ly-2P0?l}Kop9;k=mg!-$_Pz%)k;d|5>SS( z!n~Ovs!%x{*%?WuT4Qy;Gk@Dic_-PQh=2Bt8s}rA3pV~$)O3yItU@B7>)iIMg+Ko2 zSJN3&i510Zvi|ZK8Lll_FNha;=IWk;b#=0BImBnKx}9dlPN04oRuJ_44V`S1?r#CX z9_vEXc##OPmk|3{_m$vJNImtvo%~+{s~Qu-%E_YyWT0CL)NgP$CN)CxWYCkIwNKw& zwcR}hGOfx`jW!MxyH(k7GEKT-`vO}kPO}Jc!OaIdz2;+}|euhPqP~aA`21SK7Y`It5eqwUlt8~Vn;qf@x_j2|8fhfGzd|pYL}U zxp!5VrN@HKC;HH8g+TR4tF@5_>%=U*pFF#cD7r#^61AO6N7j3}e=N479hw6DQHHd| z?}IVKas_Af)pHTz zunV7GUv5{Fx^voY9wd!dzZ%CWHNlOL24WrN=iid$?w3iAHIZ%(u7qs}>HV+=z1K=BG)9aF`97rwpX`fq<*zd7uVjYc9OOFq2*x3GV`y zPPCGO#sw-|vyc^kstdj5O0b#{0c;Mf2(#dE$X;C{8MjT+bgQ3_sxVydQA~kI`OcBY zIUM2H@lJ?7ZE62I>F5z#O*`OOU*VFzdxaL|D<>kM$aNfMUmS^?g;vI#c4ulkEI8c? z?Q+$u^c_I>#lscLRMPLVVX1k^I|*Ku0g6A?kqp z^ozf>KPu-X5YrHm@Zejl19iq(bhWWceBKXuwlYmCR;CPFCt}rS2f`b!di(hjd6e(g z{lOI2L7WxEXwKP%Xs9sz*Bmwx<~U2?r@d@#%=~YMX!nGQLwL38Y<0fJ1{V3l{bmOj z6cH@7NPV0TR z$!0f^P#i2uj1()t0m@8Oi|W@-IK_$Qs!BBY!A40Q9*W**MPcB9Z>O&H*wfLpksSFdy7mp^;**v9k z<39!LGaLZ9H%rgxe}|*=fPR~|$7T(9iRX}N`dG9Q+|rZ%#YG?*g0XX9q(@$?v;35C zyYDzv;rW!uGkzE-A%eDW#LheP$m5kf;#Rh@K2>Ytl6ozz@^<1<<1S3CpXsShPrJfU zxu`%xyq|aDGy<*qVpx`nKqemHY?bP63eK8_+BI*2jCeRv05bnktte{i&&F{fHh)Iz zm7FOB8TAyRLv{h`s$`6Gvv}M2%KfV*`D_?EPr23V!>rXzht@}2iuY20wvkYT$!ioi zs-O{PT~*GMLA>;&9gY}Qe1+fny^hRGX3q~-`@3Sv z?&tnlyCgBtT<(j+XL{x_X3|?}(%BbqGZ{c4G?}=#Slqc@qDrHG6bkhd9uNEM}3@~1~0bg1_SnfRaNfGUtSW=u+a}5*$ zKu7%A*vIr?*SC(hbfheXdb+x{K6Sv@r`%t`RTi7$rKy{9jWwG?`N8lmhPG7-Oe1Pg zyFH(mc1Rug-m7<;jJ23(71 z4`ofExwuLVxFGVY?|5TV0k#+w>$J=QTN!o= zVLm{21Q5)MyTsQ+wo_kB^HM?>rKj0>Uo3=|1b z|DHWUOa%3zlD^3LBTcqQHk8(MtsK#Awy1XKKP@u&-+X85VYk@6#-oL0LC&>YDH{VspM zW9&)a(gJN=R>T49n7Y5(od~fmpe{>2M2p&5_%&S64@j+NC2z!L9~rV_*UYooXVoZI z{9j84<_GFFm|z}L?HAyxT(2gWJMkWW-!Wwa;pL|GS>EERX~9r&q)E^0rI% za}94Ax;^v!kG?ZS#*ZTYdm$BN$dtRs?#%eWLl{6&Z+C{ojTA=K#L*Wsk0e5p=-Ok6 zIV2$ZkCg2GsmIdc@r|K9X1j+o;W=yyk9!86>8+p{(Y!Bv9Yh>IYCV#5}10vj-x zu>dbELI=G~V?r8YjF7?Jae!P>Zcf46L@wqz%3s4j$DuY$PSndJpt)_mIn}$`KU(Wu z2U7?lvIGvFfksL3?)C=rStKA72=+0~zoL(U9%%Hbv$Bx?wdXL;2GScc_{haEJd+Br zzdF?Xq?-=>Jq_L#{|!X+zSoD{bz(ivEODCNfT66alCV(B z>cZiS`=2Ye!}V&wL+x--IYGhA!`60P4H~;j)LD%#8yP@SJ3YeVq*g<@9Vp4?>ug$%a&=1fMgPy@^|Xdkx1q`#37m1>3m>4tuw8xTFpd*9r&1%)korMZ?EtjVcTj z5a#W@#WvAu4p=Jl9WCl2m;|v{`;q*nTU}RNLjOIq)KO^rWGw{h_4AEv-*AC!=@1?F zqraf108!G+0c$Xl#qZLk-{L(k^m|A$?0sv~-sj%B{f-{0Os><^)oab(!=E?$;JWqc zpzvm)Z|Z)%RA=6sf>(r9qy#h!aLrkNG+V0`sI4|^m3B2G^lcuFTK}r*rhK~A)#1d{ zqFlsl^H(60rwi3~yO>|RF_^8SGy$S%mvs>1#|5%y6oU}zJh6oi0x#Cs0<2%n0~JG3 zu|}M%>gxE&n@A92B;EBq*Sc8nR#x7Yh_`@7Z#od_Cb@k8aWDhyX{uWp7SN8v(_hvg zlBX8LA$4r6=N&v9sk=IEUK{V-Y;ghMq%&sFQSI4T`#{6dF9B!cwd!C!@IT((R5rHK z`$+dN9?Bh`-o56_`3Ly*n}1*9szMI*Y`-?Wh2{T=2Ql6X~jwH(dl-Gz&h|6*Bi&g^&V8#K_(7iFIf4rw4A6rvFyiRhWHGL^luKHk6G+s9BY__M7l zScqKupxs`{vjpfA8RDQozbq4eqn1%q^g9iZ;U)1MNAj1tK zLX5*$oB1N`mgq<)8HT}OJzFG|t#t)*og?|c@HU`7O=4;0FF1{66?Oh8SL3pwR1{?h ze)^dD0925(GnQ?PKc|*t-=ojUW$CdX#_i$JWYIT&(yitEX~e)g3+lrXc04#P03H5U zKEUgy%~RO+MjTPD*T$K4P~9$5-D3?%dCFS9%zbmQ5pZiM_(`4LZj%^9{TGnQ3M01p z^HP?PugLG@9Z>9dnRUhrj3q}v&9A*FxHT-AT_XgUV(U+TTGSluraq6_2NY=o5EXfp zgE1*bQytD?W0x@Y?#{WDZ_)iQ>(Rd3n=1x=WW-UR%481$kt`D=-&dz95?O31uDmNO zC!okInzbVNv^;+Ks~}#pFu(IA7xr|H-q8QT91YLBPx7rKYQ+JN!)Cxa?1dnyBqmsJ z6Fbr3)xsG@zU55qD@)yadpr2Z!&Nc^eC)BYblVS#2$L7RJ&d;-FdFflm<0F!toxUYt#LXcNW{)yq1Dc`f{(o=bqQ?yJ9N^ zoM;7@Zz#FWyrkl_vY!DNXw9lRzoF-VQ!_0Bet8fXhnvaQ+)?G3VD88ow0fLxv-9p_ zR{)QRbJm+_NSnL!xn&|~l<7N=Dt>ofJRgv|+(b?ON6y67r>l#O=ECa{mJuNu1R7XJ zT%+4M!~~_QPjp%(4P}FLDWhvHQ1p~Pw0(MevVxb1h3#3&J<;m}Nz2qPL8!}-?l{GF z=ka$_d*LHb32-)JK_x(dcHr2vZ?EzOhe(R?{VuaPa2lTX6QrdiG2Fv1oP&2jbchVY zVZq}a^S0u^l|qXSOb`>Vbe_GH>W4}|O;@?UJE`t!e$MsO2UIExZZn`A`DZ`i(QsNH zMgzeKX4@q*$=%K3UCWWq<(FZ2m*)d@;G+=Ufp2SLCzI|FL;Eca0gU3~8>SJ1)qe9?ADT z+KH*;eSsUqjMKiFoInARsoEyV$4u$SW+oP6D7aSk5WeTQ5`w{15B)h^wg7Q~4SMusCzIau>+-I2zY$dXV*Mn)+!PLzG-4qUpbervBDEf$jFYcZlRFhuZGCMpib+7;7Qn(?sv(FA%&kpqMt@hM}%qz|-iaWKwJ zzw;^SbCC0*ii=b0GwqRn(K(#^MmR)<5;x^Nwz2sgt@0Uy7FPl)y-BEu>4T`D<*ew- zm`*f(g+(+&{m8lqCcd}6`1C0f%((yxx>ii6R6+w{vYQ(nnh#~ZC%$+24Cjg=Q~?Xy z2nZ#>IPU*B_eLKWQ9>--h*$a+;jUG5HE@z|fE>Yyq-c(+2weD6K_VVnoxUFwX_M{{qv zUN};hlSp)dMA=q6`51_bIjV#aKR&lY$|qWW@Z~L+yh#T|Zx}o?exg}+fb|6@>(s-4 z!{b?k8&cz}reFuq;S zp@kqLsHj=9#w{tN0f;`#KUB2c3RKqGNY`Qu<+~S3AS~k{#*ZmFjUAb#YuDQw3^&Tp zhgjgyEa)sSYek5cM#>P;AhBo}9>06nNFOBlTJ+H%F?LqH-K^$IpBK=YC@T2KBA7rm zdYiW(FYf_*8!jMA(iS!z@S;9=QivgmTheV^TYz$IQtPiM>;z&BL zkF5p!C3XR2w}TIj+0|^?R1Yn zioMyvxcC@*x}B_LSS7h_j7_4BbFRH|3qptCdCu~IWVv`8om=Xveo&pm`N49&1-AIc zp?xm*s}w)1dKrS;r}* z5^dhPjE0%&?bJP87Y{!^n*6B7BpJh{S$(a^JT>4E<lG8?QHcGuIIlOXd~S&1N-`^}(r9hmsa!(LggR7gnmgMFUzZ2w4sgcIC1G zdo=c@L>!LC5y}YNOx5~a-X@!6pec3wg0$<`*J7!r5S08yXWfG1p5F7Jg^T1W)9wrw zy}I+=kvdX5D}zp#P4&N3C?&*x19h-`F5v5{g!6Rq-mpe0V|)yR;r;Tf zM)d1!^`i8N3OrhnrnW2lAK}mTG^xr&%oBBOIJDi=?bfm(vMxSVkRfW5KxIh2#2<3D zx!DjC{u3_*J)ctg5&t+255r;;HO#i9wMV)xqw$%FvXkS%_rkKUvv|B8Z^V9Ofm*1? z-?&u|n69m-6D07+MEzC_ZyBZH`HR8v$4o5CbO#T<-A`g?+mk43y}KT~i+P^9AtJMX zEFFk2xlpO&bY!vdD(Pb96PMFP7VpWQX>!{lgqYq=2(4C>H6S{rt3F%B6MKP~L5nhV z*W?_dZp+|6UeR;-)F;qH*9@*eU@}BmlU(Fe2kto!}R=y zZe-E_&l;j9NE@g1+yqW9Ci5D!FPu3D={>Cx z*qpiLd1SgyWYE>R6@|do#~0v8l6U{oC`gAE6t0c4K%0PPExgtVt;0dqgZ9^7N?th) zA+#V3IH`4&XRDtItc{?+^b&Ytz;rq9tw=(q%D_%V+KN&oSRD`Cci0b1W?v#!sUn0{ zNCJlzO`X>YLkGA(8ChU7UPIthOl7HgH(;18ZOaH(DBBv6=`QrEK6>0-AuR j(z!YbA#|nZz<*}bd}T#Vx1Lj=V~9Lm{an^LB{Ts5Tl7zs diff --git a/mkdocs.yml b/mkdocs.yml index c43d655a..9c8e4ce4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,21 +31,17 @@ theme: logo: assets/laplace_logo_inv.png features: - - announce.dismiss - - content.action.edit - - content.action.view + - navigation.instant + - navigation.instant.prefetch + - navigation.instant.progress + - navigation.indexes + - navigation.tracking - content.code.annotate - - content.code.copy - - content.tooltips + - toc.follow - navigation.footer - - navigation.indexes - - navigation.sections - - navigation.tabs - - navigation.tabs.sticky - navigation.top - - search.highlight - - search.suggest - - toc.follow + - content.code.copy + - content.tabs.link palette: # Palette toggle for automatic mode From 9c2124c8865c874400c52615b78688a679310aed Mon Sep 17 00:00:00 2001 From: Agustinus Kristiadi Date: Wed, 18 Sep 2024 09:15:12 -0400 Subject: [PATCH 10/10] Make font size smaller (too large for desktop monitors) --- docs/api_reference/marglik_training.md | 3 +++ docs/css/mkdocstrings.css | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/docs/api_reference/marglik_training.md b/docs/api_reference/marglik_training.md index 5e2e7f4d..d66c51e9 100644 --- a/docs/api_reference/marglik_training.md +++ b/docs/api_reference/marglik_training.md @@ -1 +1,4 @@ ::: laplace.marglik_training + options: + members: + - marglik_training diff --git a/docs/css/mkdocstrings.css b/docs/css/mkdocstrings.css index 27a01b9c..136db40e 100644 --- a/docs/css/mkdocstrings.css +++ b/docs/css/mkdocstrings.css @@ -1,3 +1,11 @@ +html { + font-size: 19px; +} + +.md-typeset .admonition { + font-size: 0.7rem; +} + /* Indentation. */ div.doc-contents:not(.first) { padding-left: 25px;