From 274ff1613960899a750cbf2390d158c67b55e496 Mon Sep 17 00:00:00 2001 From: Zejun Lin <871886504@qq.com> Date: Mon, 25 Feb 2019 10:44:42 +0800 Subject: [PATCH 1/9] update HowToDebug.md (#752) update doc HowToDebug.md --- docs/en_US/HowToDebug.md | 92 ++++++++++++++++++++++++++++++++-- docs/img/dispatcher_error.jpg | Bin 0 -> 22964 bytes docs/img/trial_error.jpg | Bin 0 -> 53585 bytes 3 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 docs/img/dispatcher_error.jpg create mode 100644 docs/img/trial_error.jpg diff --git a/docs/en_US/HowToDebug.md b/docs/en_US/HowToDebug.md index f64adc5ef7..e33cc45480 100644 --- a/docs/en_US/HowToDebug.md +++ b/docs/en_US/HowToDebug.md @@ -1,4 +1,88 @@ -**How to Debug in NNI** -=== - -*Coming soon* +**How to Debug in NNI** +=== + +## Overview + +There are three parts that might have logs in NNI. They are nnimanager, dispatcher and trial. Here we will introduce them succinctly. More information please refer to [Overview](Overview.md). + +- **NNI controller**: NNI controller (nnictl) is the nni command-line tool that is used to manage experiments (e.g., start an experiment). +- **nnimanager**: nnimanager is the core of NNI, whose log is important when the whole experiment fails (e.g., no webUI or training service fails) +- **Dispatcher**: Dispatcher calls the methods of **Tuner** and **Assessor**. Logs of dispatcher are related to the tuner or assessor code. + - **Tuner**: Tuner is an AutoML algorithm, which generates a new configuration for the next try. A new trial will run with this configuration. + - **Assessor**: Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset) to tell whether this trial can be early stopped or not. +- **Trial**: Trial code is the code you write to run your experiment, which is an individual attempt at applying a new configuration (e.g., a set of hyperparameter values, a specific nerual architecture). + +## Where is the log + +There are three kinds of log in NNI. When creating a new experiment, you can specify log level as debug by adding `--debug`. Besides, you can set more detailed log level in your configuration file by using +`logLevel` keyword. Available logLevels are: `trace`, `debug`, `info`, `warning`, `error`, `fatal`. + +### NNI controller + +All possible errors that happen when launching an NNI experiment can be found here. + +You can use `nnictl log stderr` to find error information. For more options please refer to [NNICTL](NNICTLDOC.md) + + +### Experiment Root Directory +Every experiment has a root folder, which is shown on the right-top corner of webUI. Or you could assemble it by replacing the `experiment_id` with your actual experiment_id in path `~/nni/experiment/experiment_id/` in case of webUI failure. `experiment_id` could be seen when you run `nnictl create ...` to create a new experiment. + +> For flexibility, we also offer a `logDir` option in your configuration, which specifies the directory to store all experiments (defaults to `~/nni/experiment`). Please refer to [Configuration](ExperimentConfig.md) for more details. + +Under that directory, there is another directory named `log`, where `nnimanager.log` and `dispatcher.log` are placed. + +### Trial Root Directory + +Usually in webUI, you can click `+` in the left of every trial to expand it to see each trial's log path. + +Besides, there is another directory under experiment root directory, named `trials`, which stores all the trials. +Every trial has a unique id as its directory name. In this directory, a file named `stderr` records trial error and another named `trial.log` records this trial's log. + +## Different kinds of errors + +There are different kinds of errors. However, they can be divided into three categories based on their severity. So when nni fails, check each part sequentially. + +Generally, if webUI is started successfully, there is a `Status` in the `Overview` tab, serving as a possible indicator of what kind of error happens. Otherwise you should check manually. + +### **NNI** Fails + +This is the most serious error. When this happens, the whole experiment fails and no trial will be run. Usually this might be related to some installation problem. + +When this happens, you should check `nnictl`'s error output file `stderr` (i.e., nnictl log stderr) and then the `nnimanager`'s log to find if there is any error. + + +### **Dispatcher** Fails + +Dispatcher fails. Usually, for some new users of NNI, it means that tuner fails. You could check dispatcher's log to see what happens to your dispatcher. For built-in tuner, some common errors might be invalid search space (unsupported type of search space or inconsistence between initializing args in configuration file and actual tuner's \_\_init\_\_ function args). + +Take the later situation as an example. If you write a customized tuner who's \_\_init\_\_ function has an argument called `optimize_mode`, which you do not provide in your configuration file, NNI will fail to run your tuner so the experiment fails. You can see errors in the webUI like: + +![](../img/dispatcher_error.jpg) + +Here we can see it is a dispatcher error. So we can check dispatcher's log, which might look like: + +``` +[2019-02-19 19:36:45] DEBUG (nni.main/MainThread) START +[2019-02-19 19:36:47] ERROR (nni.main/MainThread) __init__() missing 1 required positional arguments: 'optimize_mode' +Traceback (most recent call last): + File "/usr/lib/python3.7/site-packages/nni/__main__.py", line 202, in + main() + File "/usr/lib/python3.7/site-packages/nni/__main__.py", line 164, in main + args.tuner_args) + File "/usr/lib/python3.7/site-packages/nni/__main__.py", line 81, in create_customized_class_instance + instance = class_constructor(**class_args) +TypeError: __init__() missing 1 required positional arguments: 'optimize_mode'. +``` + +### **Trial** Fails + +In this situation, NNI can still run and create new trials. + +It means your trial code (which is run by NNI) fails. This kind of error is strongly related to your trial code. Please check trial's log to fix any possible errors shown there. + +A common example of this would be run the mnist example without installing tensorflow. Surely there is an Import Error (that is, not installing tensorflow but trying to import it in your trial code) and thus every trial fails. + +![](../img/trial_error.jpg) + +As it shows, every trial has a log path, where you can find trial'log and stderr. + diff --git a/docs/img/dispatcher_error.jpg b/docs/img/dispatcher_error.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d955087d9b3f8091955fd08e048d91704ea281d5 GIT binary patch literal 22964 zcmeFZ2UL??w^o;a0 zG#A(|Ffy~Sva-@Luye4na4@m3viyDt2`TXzGV=4}7TiFXGR zuLER^Us_9h02$CN(Yn z?YoTJy!?W~qT-TIRn;}Mb@dI6P3;|>UEMvsU%!ovj*U-D{`iSpK%p0xmRDBS*0DRg zd;156xTE9We31a8|DCPBarO^89zsoV-IEl z=@(Qicam}{+s+H#FvPGv@f@aR6OviDjQ!2pADsQqF&6gk;_PpX{mB;+pd}?CE*>c( z00NGggP{HdZ+G`?zJcTzz(DH141f~63VB` zq$KZN;!6W(+S2>SIH@AVWG6xXnpc`OJT=O{$T1WEL2d^GEPi!%ZWi<*z%fjf-@2pt zcF}q|lZ}Oy_JNaI`30eaU9HfAeF@V~crD@JpYhk+oY4?J(Z^_MP^tvK5fW)6*J7{Uh>;ljJ%{Fwa~`D;PB91)$#pI*)NS%3c=DTJNmH-NtwS<(4BNwEn+OD5XT zV6IeLNiSKCL%!68JBho_+EE`j>fhPtX3lb^-BbBf{fhF}(~!&Q&c)`MD;C!Gs_$MI zX-ZEnUGuzoc+R9U+N%rYxoDId_gLd*=kXySyiu1j8Pw(oh0JU33nNY6`+l&6XE4pJ96`Nv3W8U%F z6oirfN)N6$QlXIMP?FHa@l$zJdboEcN7?3<#o#yVScd~=HiLvIxfL0bUWmc{`N{oV z@C)G|u7yfLR2AHH z;$>5Gi>kvjX~S08>F_xH*MbMM02h1TTJK9!{7FA=vrA3}sc7)j8x;-LxiqgAdrEZQ z)IE|WTC^>2l4u0$+V)3q_65OKOkZ+jD!YzJQ}Xq;mkztXRNeHrR|+m;t)W`z5#;-W z)`bsVhNp#P5s9%Y)hFt4lcu{}&df*G1G3jyc&W3R6O||Yx+>KJYAiGgrEfLbsW)ABeaZSDzdN^y8tD_N7 z=6QsT#0;pLo*Wp~(K6AUBghj{qX+p`4ukb9qV#bfR5KszdedBF%VUbvEA{Ryff<93uJjdWLRJZnNY*`u91$GDwVx7@ z1c|j%B;(UD%Q?Lr=<;(WycsqV!uNSRq11y~vwYCG1gvkKKl0N;^fd=R$7URQUv?95 zzT9HBU22+vje)aGtZc}|#;|)zq5Zlo%IN({c7Z1^Cr$~4IOor&Hw;lhZ8&YsH)jP! zm+zec(xUk(ICE_#D&<99a0yNWGf7}gj4Ch7hg;v-IoQ&kdT00Y?u8_5>OIOOgBJw< z0v=|)DttPg9cqJ4gMs9a5IDXOrHre&uiIm;emwWT+M4yEvv)kVzDZ7lcy)Vx8|vhD`eb*2Ob`Lzx}L z*p@|#K*XiiM5__|(1I>^L$UdnuNrDq{yJ7*ec{qm|lkv3%&2JxCVwInu(N`+-z)iCIiJeN> zY&A{qed{NZRtdVgL=0iUTD3zS*hVNWvk;W;kmVCxtFUGOkIZ~ zciQ+bpTN{(YvTL*K zxIh9A_AnNrK3V!aW39gHbF6dJcb%5HkY}gl7FBpxoaO%I4QTrEHZG9YW5Hb64_nBi z-Ra9K<4*1D>?dBAX$N-T{!-=|zmv9@tfAF4g1>=EWP*6$l3ep}1$eonf=sLF+b+pp zWwP!gDYly$r*%~#eB}_75X%LwO+y9>+&3P<+hYkws^-`bX;AvWB|RK&Ou^AE*lzYP~=*1I~u>?XY2JwXfv!RCg95fD>w|IG&tCsk5SkTc;+AGmw5pRO&LD9ELR2NP8 z6F8JrN}88vU1jaFsIryvGzJC9LN(EK*W|V`X6Gc*ZT*GHMv-U0isK;%K^Yr@sPIRB zxv231a+^0Xf1{3B*HPO(>7EM-?Wc}ylK1pCH?;F`MV$mv>}>P=sb3eP%S3;kYN=hB z%Y}UPu@aGc&Hf?`%EuwB;pjnT;_mS%ln#BmoduKZo{8X# z86j+c2NNi>{cTaprFK}-_RmP%)?s%>2+Zd|Y1k|8H2H@Vw~Tm=w@Q@D23`e}pei#1 z&KSrLaq|$aQVe?>^_=V+ppDyS`-UZt&F$euC$=rMTzt84?O%b z$F**wjPr9S0ZVO;C2NW=doadn>CdZVnIsRtaV>Q+k6QgG1nO0-Q%qqM01a>(pmH8KL4FJ4vG!`bI zlZgGLkJ`bY{Aa_qy3qDa4D|FIXy>UrK_U@n6vA)@eAFO3=syFxMmTqQSoc=&*ASFv zz>we>aOLS4fLc8RBIM`KfMYV~V&5Fz8i4eu1>ZaasNIQEvqyCXe0_(&HGvj%a3I19 zzM~uRX8`*xKB<3YtRc!iZ?n;COd+1{47inuf0ua%#O4uL=EBc_RmL+Qa|I6zCtMcQU%?w&LJ_A0J7ZC<+hW}BGnoIC9nAyq$y*g*O-%r<(3BI{+&30Cs zHXM$e%2`gNOD>{ogMZc9|Ia&sqlB1auYXXbzvz9as$DMp{Xgh6-(Pj{Bg5O(ilV=& zwZ=c12GV84|GVf}3fOa1-7u^hds zzZg6n_wc{Bq(hXhYApIB|L58ppvKfO3Zj#T>$o3fGUQ{)+)6ge1$!Ea#F^Qa)ry2Zk z?>QxgnVwcZklM7gx|+@>|01m+LeFU)>Vv%AiM^qo>p@9w;$VWzu8{Y1hvz3qp9V>Q zvSE|fFU22gPGzmC^XtdKwfhM$y3N!vunfE0f>6T4ALq);=sEB6hbi!nCsxgz0fXCw zJFJ#YogenqzKLZ2bR+sChiAYgYSr(4X8;IsL}ke78B=s%i8wkXI#PH3_6!Tjzwyr* z<7U6=p}Qw%!0QNE*x$whT8XPfnFtBQ5_j(iA6x1vD_o&_)GPxW95IHscBgC3)EpE3 z`7uD^Ng5TC;Yi5NJwb1m&{D-^C>!J+R=6XZ6{3i#t&1x2NAWyf3fr~pf9VPDweDbC zu`KqeQF4uS*1r?~!(u(;QZ|Pst=udZ<}ebkT{PVxCV!A6Fn*=diYMpJo7x7ZYvdiG zA_t2*qEVv+23%}g2rtgu_2bw4^?Ft2#`I0O!xU`?w~5`mVs3Ryc5jPg`wY0BT{k$F zZuRty@6j1xi`me&I4+zXYfYDQpTswTQn!5kve^(-cxT6)bzVj+YvhOEpwkb|sB6iS zznG_=Ia0AY%2U(#=xl25hoI>fkbPtVY;=JiG3&@@W0!clmxQe1L} z3Hx*CBPS1B*v`VGT|v2Rq2W&-4XyKMNb$`<<>!S-gW3oJaI9R)LI2$ji=qg}U1}Ph zC)#Fji-D;An%yh((AF&aTc{TJ_b8YZGMqdpQ=b04B0%M}}|N$WM; zoWI-pX)EF0d)imJ(#CltX{&HDJZM6d$KTQ5^Elh2UVYYS(dZrz$RJa{W<(s{Fl2dK zdANI7rFV<@iCD(q(VpUBJDS{MWxyYVA66&ok4l7E#%fmCy(1yZtH56B=A<`yA@U{c_DI=jP{*-t zOid6>*kbhAe!Rq*uhql3FDY%(D+=DvO(Jj2CyY(ROTh*&pT0JhYG_RD8gEl3PeHzaI1oSizRD)QXSRoM2)Bbvn1fi7=X}h=Kbq0hG-R!;JhVAcf z*7&;{fq&_(x7e@2OrU>Kvj!{q+weo=zrj^%QP4vyMf$=ILu)g`>QRBDy8QH%e65~x zk|)wG<>_cW25>qZ#fJ*#jF;;LeZKWc-`X^cBJ6rv^`u+NoqsK(`*MG|8d&WkI`ycn zt%NT4XKG~lDVZv4qn_HW0&Y4^U|yjSh!VGtsAbhA)gQrT0)Sj-|2#qKkJe8DjHT-T?GoU(b_dKQN3<$m^P%`I# z(xGU*n3fI~9C@~tr{FIPkLJErq0 zVhH-Q8z0Z>K|qJ7hhE9e*-bp;+K~z9If*~I2xP=W4S=TIdxux}w&WL`&bj<&eg2%? z?^ytJ)THd`_KSx8RaZ&J#YeW7WXwPffFMqM;OV@vn9bT29!`h`XX`XaqVVHgY4&B1 zM4*=?ANf-pLy60A4TAPkJz*+JFJ-_VwuKg8j@y-vP<}C*QF`K{p&aPmF|{{RAsZA7 z>NK#Pp89F|Zn`r@o38aHDfhgW$6^H4|0+J+r7=khQ^dIvcM+ARZZja|Vdj#MHe3p`^$Whd zv>vP~$f@h!bJaK{&SkvWb3!uNRDYAL!d<3WbO|TX(WY~vhkK$IHry){q^+6wYPsqL zmr8_g*Y|xPbP*YvJKPdMM&Mn8)}%)hQFiv8=RCe-V+Eso@zM)ZzYd0k^ls=461Y<488)vu$*71tgixZDpdvwF4u+p)<)fif#*bMR0~y{QuqG`8TUwnaL(EY+ ze8w1fMFo!16drt6cU;C2sshtKU0t+_dt_I_=O=g*T^MIaujke}w<vIW~&YR&eW z-VvS15|BQAGgTt;d@&^*adp;etyN`7T`zV;4Y7o(ebA`3CVNX9}KnVpbsjU`tS zOP#Ox&N+Ut-D)t@r8-LFwx_idR(2QzQ45{ssY+^M@Ba+Wxak`Ix;fx3^AhBQ%uxvK zzGl6@G~0Y)1ku90AM&>|#%y<-0dzJ{r|$u<+l$&k&k-I5(jf*LDQ!2~z7)4eViF(V zqOsr6@!8QetPPbdC(;%+))qT)PxxD;89PW5j1Nk@3*=hknB$jG`8D+dYr{8f=0f^D zDZCvj5{@gmY+|J4CT+EGSh-|IaNs%KSzD_;VfSoQ=kyp)iQw#VT8n(Kq# z?m5s6jP{HzPL8k=0h}tg`U-QLtqDdxdUGqq1XDX|-sR)LEHXWH?%uOFbq-p_Hf7iS z3jYc(8i%N&Z(C*cb9U$(C0ge3&RR%#Mq!iEep5BVbddeD94={t=fp`uOP$>pH{?eP zGwoD|An)u=pWsy51(UWDT19sy7u;UBV(+4?Vd>H3`%}dg50b5{oPDco=ak6BUx(h1 z(=!U6o?JcLL|_IDahaGaJ3Lo7B`WrU8usmLBm9)VMGGhvZ{5-APbhMul*RP?GAg!M z%dXLgDr#wr93fAb=X87?1M;33Wc^9*SMhFs%uT9E0Yr|!GXbZ;s}UwJ{ioxlKdh?) zysjl~GE%lBIPDIHF6rGJB|8Utf)_@KLfLU<>XWXl$&@`#0)W9qwP2*xzA(a1Ve?o&03hQtWpZ?0ZjfCKN87s-u{AL;Jg@I`rt*~5AKhb zq0n#9MxX>_cV-Lo(_Pb)`;+j@?{TkPdbw_q7C1<27T?#0REF=$x{SXTXEA#Xxr0ql z%*Qns;BK^~hj5MNW~!x|WUW|_x=EfFRqSRYGh=-8xOyN^pR!`uN^PY}hrRuHB6T4c zYRH6xulGynq@@>GeP5Bla(%rh(0hkcdGW)?rG%S>5Akjo(*hq@2St^Zlk4-0g0Zxe z9#{LJ*Fxrw_KbP7VI3n zvDF3g(j+XTKE2RHA#|vmu^)JqGy>;^QTmd7 z9@32|iiF9$%jdF~XG`-e`gByq26?V9pin;Hgzu6SXCZl04bQ$blX6};QP0C7HM{ic zj^&z#3}7|@;!|b+j;e^WU~7e9vI%V2QJZ@=%i<9q9w8hXoX0y{n=(KPl&OOj$BHq) z4wN1TG9d<^Ukm-;?UCj=pI;Cfy(g_hQ(HFc4;7@^e1Da@+$^E?8>kJ+xIRZ2BINHq z)%Lj5Wmd3d{Fy1rM9NCz;APsSOLu;lov&GW%^+ACOsNL=nW*qQ+oSx=~{+L zEo|nqH&ntTj?%hUE6b}_h|gkA>ne~N^Y3ow;?Dp)WD5EPQDCLlB=X_!dsqM1;QvM8 znJ6-{2<0J$?Bv=F+gZ0<%f%NGxyvRtFAbL8t*Cbi?;R%s2*=;g0ATJ6NSH`vA<0~H zIRkc2iCP`%eg^b7?Z`rp2vujm7a~xNt}Rdlo*rF9;LiYa=sXuu;E>AkwghMy0z-EO z;5MBNZgIlZiJ=*2!WKtCq}4Z3kSLbf8Sf9c@sg#If?57h*5N7~S-hVs+ z+y|u;utoV}*L1z3UE-jYpvQ?PqtLm-GvH!HDYV$Vz)7DR`nl(H4Y5s(G~)UGyj-aU zn4QoD7KV1ga7r+|J8Fq;W|b`*AE|AQG%aT2L@&g{jaDC z$-|7ExV>Pqrw7Lz6dsK;#v%4E9@?@idz=AOpa?gKA-4~aV-L_?%%Q3u20zPU6Dnag z8;3hKw$m#8ol$9aqd(|7e&|yoZ7~gy%V)stQb{~@)E_Auc7~UTRav5Xz0fDBj}Mld zh2(&!C=z_sV$C`ftW?Iyq`gpFPf)iqkSqa@Q4Hh=EcH1z_nA_BL&&37lk8?hmsSi3$J1CMQn+lNRW)*=4|O4aB4jLNUDzUPHbP5%Yu7-GEZWIX^cl98ZsHrO(MQlHICMoc@S}vw@HF; zQaB|Am{>xCV5~ccj)|7og^CUY=M-+zdIE1&ypC}mflbuPcGdQYdvB;T*32!YC0tnP z`?;JbEVe?IX$BV=E5tP}xHCCR`Z?sfz=hNA1o(hBtg2@NoE5cMEL6Mej7_7SzgFy% zb8}c+Bwe%A(HbFWzo~7CT-q$yaeQb``y_>=?mFetGwJud&oyAilP%hXxTm5wWtFn4 zFE34-Ugs0fhK$IK*}mbnKUAh)hWpfhPF%5+k7QT?Kq(Z8D4W6okpVx6Nvm#YM(?Z z658piUr%@q6i&r_L+^ z!h)A$fuMNSU1+4658@R;3TI7UkD_y-(at_U<04;&4Bo4wfl6B;18&@Zd{zDSOa%Xh zU-_(;^&0$n2fBMSGQ>gFjaXVf*CyZQrdD>hWX|_3Na(<}wID5fbD+WG_d!hK+Tw@$Ntz_7sk=?~-;Yos-<)2*<#HmF6L;eLLl zF24EX3!qsJzL(f7w?QL?93vc_ysY^{kcBk!YiC^Do60?(kP!2Zclg8k>?b1U5b|yccktmRn3F*{YvB zKw7^!Os{7&M_T4-%XV%tDGbr3JyixMg)&lk?A5#<^>$o(v>%E8ud`rc^kTpg;@{ zrDq=gTtm_-#+mxU<;QBHVT=A-98oQA)f6Kc)h*6hCyZqXF6jYO78vJxlm1x6;9R?TRCeSJp&`e6uSpTxnEFov|G@*1baHN9}O!= zRy%Rlob`WaWAx(CIzw*2+%l%41xuMgkkFgg5mCr(O4w~XXhP@Wx{0#8?M1Y`Vk2U8 z`kF$y*NvpojBOQ|%Ct6tV|e2>7j9Pr=a=7AyrJjMpRpkRBvEg4OR^}{JV zJ#UOh40e!VEreLk-f2tf%qxObTZFkUm`h47U=Au|eADXOV@3mHs?wev|Kh`&miSuqzP1(1!Xo@{RbI#vK*o=`( zHQqQ`rJ&$V-hNB1rqQRp>yXm7=oH>4HwHUP6O=8}Cgfoq5h?Hv26rjI$69m_z$b&W zw^fr2O6nK9$&fLAu&S2BKB-wiGc%R46hlN?bpPMySmBf2)LRq-zUq{ zoAxWm4_Kx4zNzn%XLy_TkpYt1LfKIl`?0=5lF6A@Vq*VPyNkIqBMMpjZfjGP%I4={ z+QZEceR2z&COxl{LRF30y5tvBBX}$4Ek6WxW+`ClT9a;C{8s_q4m60%8BRJfX)u0i{ArzJ^+nx8zmog(=f5T(uW?II&I*!F zG|n+V{4mhy8U>$N&3$mNL{O~dLy1~(QvFKJ#fImvk8B|4mql+dyz|WCPbT+m8pOZY zMr^#yK0aUZp+(10F6TrHFH#q_ZT=)fc6sTa$nRd)R`|p`M3s00ugbbwdWI?T25?lo!`lUvKvYKI*HPT5?db4zFLBT}50JuZ4wd5J75)bcHwux1cVYN<3I^%wo74db2DLt)*kf6jt z9fuZQ#^57$$C2+;LX#&hCk^LMMI!q?z6hT!NbkDn8}(5yfA!qDU4WR8_1v6#QuWa@ zbHV%L??uAHG@oel0PExB6?NI!N6Zy^8+Vi{q7+lhb5PpyS!3Tnb}dh*`XbT%kZeie zWgZX14L*8@A@{Pf?fw2MNYwJ>2IPqLE0ojsD7z(e-AtaCCI{ICaQ=9RfWPOu9*6C% zrW^#8y!h2!Ax0|LoXA6WCZ_q<&wbET)_o@Lh2BojmR*#<~$@lX8cq z>b7&w%)Z4BJ7}eTrOk_*Pw8?*`m^mm9R8L5AXazmBH9SdQ6WC-lV-{ob(6`}fP?a; z2v>NPXkH4cFijS5e)_9i+n8XIZkpekEl0QjCXX)9v{;Dh1^+f&k7Iy6{8xX|RS)YqqAhbb}gLjI(QRrw#k zZjpP2$lx^Q-CrG=PO6GyY@=EXJf1}=T$!jTmgK!GeA&1+&X{?C+~Daq9pO--{gbM1kk=voFQ=UI;7XUZDI*DPC~ zOzK4{HHUg634P0%`?#)m+xjvK$x1dYffDDq(3O?`JVQMt2o;xK_b)g2!I`@hnQy5L(>L;M%^+}`XPQ8&Ltzb)z-;JxQn!P0u5L#4{ z37jQ30?TdCj;B~x(oRb?7Hsa|{*ptMEZ3X&`X=RuKB zznmh+@i9f7P+aK}$5Qe=eR{73j!b1cGS48uHnRh1@_n5_?wzCPIwO@N=jKr6+b{e= z@Awo$`>y}w5T2-nfG|aEs?5V~-oXrQNQ(G;tG;^XAI``27jBD{)<28Cc-8cA;Jl6t`qrv!|gd#I?uK$v;i9rL`){NiRq>b_c9w7q|^k*sHP+!+zClDAZ(fY|2cU zRn=?TGa{?PSaYo&Q;~(5aga?wV^z6u8a*jL^ib|JPH=&YHvD%@SyxrA!ult-XE>8| zJ6_-9SQX974Hc0~d2;;5BG!DWE{9f!%)M#IP#}ljc8aR3vEwnF`>@vY;SeIFfmy|J|8u^%(yOJ$z1v7VaOzc%m; zFJ`}4^z?_hTqPa!*$xZgZN#``EU&%58S@L9ygf3Q_e@`1sKu< ztxy)YJAvpRf(y3vGyYIDOkHf=s=Os(xZsELP#ahdTYK0U!&*;IuOR?iQx&P1P`+L6 zgz1rV}I(TxSO;R+jH;Ju}*s z=l4^sErW~HxN{&P@s055I-25la*3kfr8&L~cr}Xz+3~D%C32cu;7}dy6|8Mzl48PR zc1b3X9BxDPxbp*k%p&NxN4y%(X(r%YBhTzpH*1E2u7u zw<|U8@gZ?IDKorfCRo|Sni`DNxLZB!-EYdCb^Kj_+(;uv+yC>u%^P?8D3NzcOV6nX z+LK!Q0A8v*IJ^06svlG4PBzR#dzPY>DFNj04}~4>+(xNa9n{oFtV%>1yu#GMo>SUY z)h*3U^)+EMrFEu_SqiB(PfBaGKXX1|0d5S@ntZxSnmH755vS4KbdU#Ig|iJ4eMbYK z*>$;#-q)?H9qRey?&%Mv4EE{C5+N6=@)((=(eA`YVr=M5F!;cD=jE(p=5j>0{86^* zR&(PG_F>{ahNC&!~w+AtMSl@|GYc&1~KT)I6n zIP>1bMC6QeF|9qvw=35RsK}oyeChfPs`|m%hiir5L9GJaYumZ^C6HiyFOQ2%2130@ zf^Uk0O&l+73-_8vlvL;?)KNoyF`M&Y^58xgbW0LS31-|9@PWedGZ zgnN7X>+v+lCY>j_C4cGtffxUa8RkE;%bwXx%IpjY%JLqMg=xRI`)fUJh;n8B|I8hK-I9ZeX!}1Wio)hkm}}`7f#09{q`QAX z!L3m7OX}w9)`AfWZN)1`{4uK^!!%Rx-nf^{uAi97#l@427i@y!iO4g0lOW@*mq+*E zUE(KbiR`$U$|rxQ-Uaz%ko$H$Zu0Iea>zJ#l)%UZipx+fD6N0lVqMlnS0)E%m^6lh z+aaj-ZB1d1-5c?}N%&dA6p?3b$EGpBk#@J}JXh=-G2-wl{E7u{PC_y9;s?WJVnLKY zpjB0n8HD#TEyInznO>J^EVDn8?JN1u_~(SS6T_cB5*dhR-4K`FcAe}xEb|nu4EDNT z4fseE#9EQ>^pPQXzvOW(ZW0~PVwF_i{$qdRZm|@oIpOdZTlQ1%xFE$C@6nf($I$eo zD39DJTojt3^t2hB4dDEGhH+88$r>q?22?!oe;@*@{;(IJ}@;EYBS&*~X`h%}0GYsS?d0W`dqebxA2Uhs0tCyYh`d z$2e6My!0^M6vA8YiF0s&wx7uRL-mtH`SlBJj;_m{_D$`j?&iv0s3SCH?`L-+)1%y! zL74BBSe3~LhyyH<_D4+!JysW5<2Dm|WkdVv^qfmC|EKQ0B|D1yaO?>$F+~MJzI)t& zV=j-4Y>GTDRLvw?lH_P2ufLd+xI#~=jXNUCW6Jd<wR>AnHRZh&&(p|+-xL%5MYNd9pa=rxPzz}=M$&Lt=Et*rS9eV?vG|FTczotb zo4tXDgx>>`JnMq5ND6k+m>;L5c{gVL<;1}znC`@R^UICay6El9j%eV01 zbB|Eg@uF|&nZuT-j+uz!sc|0{l)FxZyeVt6`@x!6M8xuVt+Q)){L_P`dv!2piK&MR z&dW>|73eqlv_{wN3dB6u?W|>| zZ!|ioOeJ$g!7Fk@RaFtLGX5acFwR`OAS!awaM(}J5iEA>=}a)@a}M&)O;$OW)kUc_ ze@a?@v{+e)gKCV{3`W7lnlb6|Ej$(ecfhUE;-)t61=+PDky4Co$~37vv6x zS@y`~e2R9jb#l;O9i3(t8h#yEETK<|7a_O7riadB`;eY47Yp;)Qo+)Law#s@g4SI@ z(Vi8#j!PyEbhSJtO$v{@y!S}36R`_?q<1mtx|ppOcKK`UQjk0K^%G?^vf4^--@8Z$ zXez{lOLoE{=;CrXj&xl7(H+S+`W{q$lrkyUZo!pe)|zQ2h)ao0&a`f)k$r^5$wsg? zH&LRpiiwnaF*&4?cy5{Dz;;;1S{2=yZo`JnegRQiIErY2{+y%Vd;GWnX(h2UaA#&( zV@0vbjoNrY|MN|Xsusu`oh1qyvw-V9=6dkm4I<7St=z()oJ#MFy|j!Tlx+Tbx&3>%|RwqhYm0t?+MGq0VgNI~jzj!<0>o54Rz0}P1S zpTtae9Q$5r^r4#3UwVMzzjpdHV{9F)PgF-Fips}t4ojC_WaaCi+Bgdv+0=EjQa+5{Xjsmg>FR+^x^izO8#cd%-=-hgh$;aJQ~B$3^OE?pk2_dmk<} zb1?_|d$%Rpmb#a;0d|^6cNUVhEY8!91YTTyrw#wm-WkBQI^v>u8EHP*1_r93i$dm{Jb$I@3Taz@^%cqI=2g!hx7S;7u>n)_7FPfqw$=#j<&j^T&FK%D5}< zz@eqolx2qmjT+dk;2gI5qTTbOEuP@dB%>%Ahj;{40vz$ z%D>|vqjJPOzjkC3d77u-7$5rJM2Zp=+SB-v!6z4Pe+ERt#z3pY+Jh8g4zj!WAN!6W=^} zJu$~EbVVR=OcArZ=`KrC>XWkB?B|$xO^=9IKP84Dpfliz-02(vM#zDu=2)c_B40!x zMiD#IlV;qpGA^rrqO@UGNygLQ7f0=|VsL)TXD&My;<3~2Zja3**xh_jmxUDuRltV- zUD>~5B_n^~Epw-xEy8WY^KneUkq1FH?^7x;wXe=|dy;xe)8FngAMwrB0_%^4@3B)N&TxJ=`cNXBBr#I=$8E( z?AHzhQoA$#qGUKe(@&~)v`p#EeCSm4>4n5@xOPP$!e#J?-Fy4^IFWctG6U}LkuF%W z?f9S(eDFc(-k1+fa2I65Qd8(bLY@d!ztPxU<|f9ykWC;8gaqdFL}fBd?diRXZxCo7Kb<-`~|;BBCFW#fi z)$9jT_jEMbcE;++l9{^N6%HhoDv4~{d$J(<<5J?M(J-b%OY2NqA~xCEAn`Q42<^VB zoP44-CiA_srsZ>EP1Ey&M_Ftdj{+~`+ppB#_5sVLcGev-YLa24GzC(%s;)Oo+U zdurB|^$P2XXl4D=nKJ+eJ0hG*Y=x#TPT>*E--|2k$6NRo`YCq1M|n6UP$R#}9BS*6 zmmN%!^_!zFbzIPPoi@F=_EPRCT@Z&)5w} zaC#YYyKJJmAyJgBbEv|)ve!5yvbOjmt!?!ju~cbu@RZ<=&HHuuyDW<6>p?T|U`MAs zr;oJme_Y@Huj~H-^YiJja{WSgRqRR5(r2N7{;(TEyVmr>CO&F>kJ2%OC!xZG!DeD1 z8S#wBHkt?z{5*v2?t@P8?pW9+F@0w|{A-_q3Rl!)WN{7a6WQunmr*!;kmd(#Ob37a=cL^ao$nouQjs%UN1`E$XF!|Mb)1hSDz#SAIoNu0+&Qm#a_E5J*9qe2 zIGaF_#|@um6HNffh-kZsiPVd;J+)kF&ph6O{v18;M0J}GQba5lizmpo))>_2Q!pF^ z&*4w?h=Rg}clry}?yfAZEF|H7eJQA#xZAI+Lr-vXXv--zE;Zm0@js9dm1l753^>Fh zy5BbiPXsjmLC?hUusS(lP+hrW=>_QvO!-zChrn4E)2ESzB3Zr_QG|HniIS_LF^zByHUW$_+)$5zRVUPMs z_a3JtJ^?Y`;uX#XPtNkRfLj%z)zdQW=YwPBM9-C7|B`UKum?8VmD`_R_G5JUof|Jt zY+np@l7wZk)HVQ2s8(G zoFg1e%&kN|!@3`qA#NgYGk+N^XQN>vlbbtBdcpxK?f-J_`UK zR2=V7zj+NVqQZRI#pv?1r(E9kh9>RUhffnUNyPZ=W+EQLQdiJmCon{keu>rA}N_wBa5DX!&H|?k59^uP6 zgtG!C1I_^2pQyQQ^AIm6mRR4Fscd<=AvxeqkiWGxc&&VJejEQLSdcIUT}>}f{QJrC z8SwwWwf z-6Yny#@J(#y(tYHOr1PEmLXALr9={)s%+7UlRc-C2idzz#9bsL9?(;}eee#n@@<1~ zdQo*$7=L69oy)t$JggsFsCTr}L==DZ#2R8fjjEaL;p!0C9x=*JAwbFz+rxSPDPC?e z#@-k4n7gL@6D3-2w_IpJtegf~*Q~K(R43k?L!+TB+`i6#_$qi@2-~tI-d2@H+p1syXf>P49RY5=XSJ0eS{GnF-4UZr1lIif{FQ22>M`&@~%K1i2u2wOZ$OA zc%D)OMj+dplhVUGh_k3_&ukB$;HYb;Qq)ZyU%17z_VqjJlHcX3gP^hjfmUu3*VWaX z^PFDB<`|e~y|e38Tn=(LADxG~RVzy9x!Jr~LGsI(C3XYTZ8RkB0Mn29| z&GRrbwz+`^EtZ0b@JFEsSW7GU7WP5#hE-n`f*T#|j63>NU9lyi*Z7Fu4Ak(FS?WRy zk|O%5rkfrVpZ`r`l}?sVXUS<^TQj@dE|cmPr!Xc8DF`mOEefXJQn!jkE6wYdWC30J z8ML_^g4rB>J5OPnUfD+k@q3+Yw)ScFLuPk$>)fa#nGY*?CNrB**K(sP)8M4I6vn{% zULzIG@U0L(?yj*%C}Tnc@{6fY-Mbdd$exl4inxTu@$U*CZ1{#bG62Fx1E9?6S@)@Z z7^j`d+2cm>1ub+X|Ywn0mm=#pV41K3bN~|SC0Mw}PNt+#B3js7! z9<^Q;>|)WPUO3vWN)*gH8S5{kiGHL$&awyBFDWL`b6GX3e%?xL6^=$9C0g_BYsI|- za#xlhEH>%@g@JQONHz2;=o|rhEhQ~A{@F(tsF~gw4>@rR&{K?D?TCNv{Uo|-1D=iC z8@CovZ+c_^*rF<%+fApX1Coo>x7lt#i%Byj>~IbFFzJM{To9^9)$9VYl@R%3%k$Q6 zm*wU6W$iamv|o-@6;zBu?FILRev~)mU!JUP$x|7JaS`1wHxSH)t1y>=JDe52AUGk$ z>Nr0y(cv-7RA6p}I@jZ}l^@AT7u?k%ox?JSmjXs)ua3WoW-fNLNP|elw})-2N+YQb z^IFXcpZ=7rO#ZHS5N@H(c-@K`PuQ3xJFUo$_LY{3(>6bJPgdi@xIhNa6a04KM_+c+ zU<*oN&bv0(3OGG$v9p$rrZBf6Mof~7WWXB|O%Iw5g{x zEax9APu_mL845#w+u5Hd?R#0~T4Y8*H?)xo^7Y{Ybf7m7>t*0LdhMQE#H>$llE|nF z_>*qM}Q^cSmD7O<-|rWT|8cR5Midd0G5o?bAFV1z<49p7nfi%=-FIh;*>|;38xK zjd|m9%DNU?GdBUxweCMOG?%wy+BQb3CvR#4t@+$*O}}T~_Xo#3bpv znuTvW{Rh@&C&)mnZ0(3-Jzl*RxZv!JEcKqhG*~B8?zUu6*UOGD_x=~pSok82p{)N! zT)Sn)E%GY)11AE4?OPO+AAVAE4WngwZjrzQ<%!Iw)mN&*h=kC#w$|1MP0eMl@;!#u zmX*QTzcGZQPDVWli}@9YZO6kKO#x|%`p^Pkj|3w_mJ~M>_jnzZj*awOW>jbPlm3`Y zy#{^0u|}1hkfqVDRTde_LDh_(|FT)tJ5Jtx^q({N{+&?%)7|*LYKI%VYg#nOe~tYO D+6AP& literal 0 HcmV?d00001 diff --git a/docs/img/trial_error.jpg b/docs/img/trial_error.jpg new file mode 100644 index 0000000000000000000000000000000000000000..311900d3cc29e51d00f794370f0fba46f8be8eef GIT binary patch literal 53585 zcmeFZ2Ut|i(k{9X5EKMOa#VsSK@y5k_3by=QJW2 zBqtei&N&S*;jVA*v-iK<=Q;bo=RWuT;oNVa*MzlJudeE@>Z-S@oA8S;16+QhpsWB8 z5fK5ez(0U64#)w-=g$4{557pie^PQ%QW6rVQc_%`ptwXubNLe06)Fmf z%k-D8T&1O>qobr|V5FyIq@ksw{i6~hV(=ak(hH=d7ice0T%!Hoei6O{SICJ%NH&Ox zIDvClh={Kc5t;!G03ad-Tl+_Y|Hq5y9N5P5WaJkvQh*nfUk1(*5fh&yA^xM);MLyX zd4S{!Db4k}56@rKFec-)rxku1okq^}sJM~tS^p0AJrf7N3m54b7@3%PZt&i`#U~;v zCVpSyf!t$x1w|$0C(mDKYH7dJ(KR(Qx3ILbwsCZFc5!uc_waug5Ev935*ia57oU*$ z;bT&IMrPL6?3~g&_x6g|COx2Q}$1ET>^y*K$ZH}IdsyY=(cgqxKKk~S`7^O2!TFMyZ6<{0sjd{0(nLXWE3SsG;1TEX@ zkwRu8)O@x))JghaTCTc?@w5g%7+Y4uAZud=`oZV#^K`KUfP&=*0XPt`=D^$uBLJb_ zPWj#5U|1gb^kNjH5?5I45_`6&Wy+V>T^pf#h_hpHj5=8!0r=6ngTh^6R>V_s5`eiJ z0uT@}i#TN#iNZ3<6Mz=&IQMB%X*dDco|v9h{JisgWEzj_#?umQp8Dk4<>@Lqo4e92 z=6@U$txM}KQo8@@jXQ}Co#|ibE0Ur@bGaF|V_fZy#CsRulw{NJY0uR9n#vp^uU7mn zc4eRCtZl!*ZjK@HU^@_hwQltAAs2ca&aQur!NmgLTX_2^Ro5bUzE%)6dx|NADx`F8 zBpq@(&{aKqLUYgm{lj(T4(^2`y~yse+Ru;=(^HVo?4}4eyz^oLgjHrd+e^9z-!eu3 zF1&6POttA9dvl@3#R~@+D@dLri{jbgL1y;5+3D?apxNHHrK6e90+frbZs(H+j#oDK z@gEZFD`B7Q{-M?d!wb0dYr9jqFu3syik7ErFxhp8yZAhsp2O2K@VfofXD<~X!i zh_|<$saUpjogkmiI3oYU+s}DhRvx!4iv$g#GnT}Uqm~u_Qmy66gE(|+n5nnDsaUdT zGe6hZgz3+B(J{y^{!~o%E-{`iJAg&^jO-}kwsGa2)HXlPG@xe_IVjP3tDY_OOQEx4 z(O)>2@fUU{A*&y*MkndYG5qCg)Lm2^;=z-f|H$ghbl*Qu{>cPT=>BBF|0zsZNBd9| z%h1|hCICi-1mKIJD$(4qqCrh%ao-_Da#h`#5_}=X%5Q7x zt+H|C39zgEEv4A~#^=nb$rd=zLXqrVR-J5O+_#v^L_?oUcQen6g%wzfr(D1fl$TgJ z74l4Pv|h`$>QO-NLovFk)QA-waP9%<>lIaYNr#W$5rA^@f!pxgbzfu89z5pM51+LU zUzi`Pe`B;$RZaj-c^@p}iIoXJv-)`>tROT9bs*3|0DLJF|M81%>OWnq0ZwmWMg$<; zu>dh6l8TiFJjnc<4Ip|u-SJ!THm^5$rWe2WzvB;j<)bmkeHAmZXyyieS+z?Rv}w0h zHg@oFX}YFo@ks|eYfk$$c3HA)Io$)IebnZqdC3b6oEXI>?TX5z$aquUql#>}v8|;` z>tntbE-+1YNY z<)FeeHM8^RC5P3k=-dsVTzv8)_)g9TV&e6bcxdzVMUiP5w{oe+wp;mNzmzTwn$s`3 zs8afd-@o?B&*2VCoS}VwQWAS~mY+CffIB!7#n_!x9<%eLA=+!tq!FXz1YqSxLxx?A zPyBKMg4C%?l7mDSi5uq);1M3bJN_|*v1xaEoNmQ3gDjawP`a%ZmHZ zBg49u`36xTggAJxZ*P-XJ)-JOuwTL^CkGxxLW*jm^=& zSkc7jG3wi=+|S=Qd0n}vvTO#YuPz@?A^?I=&e8HN1G^>&^+9bH0bp1|m!U;mg)-f2 zo|Qo3ji&XLlujO!c&j?lz|aM`J7mpT%$dif74dU!@ay=2#O|udyL^`;aqZkuP&NJu zQ4U|RXZlu&9?;-&Z?b2DB8LO`F^km(Pv;d@x>oPD#qT+<34jHTrmfKrpkTp3u7%J3bDe+Xr|>LOl2%PUmI=nQ7`@ds89hJ?&LK9;%H zuq&24)A1!x`rflXTPDGc#qSe?q$}e!G}MMcmHi`6V|2dvcO3`~we;QvFbS{04~8bx;VdjD~5KVd)4!wC*|{1*!$v z$Ry&HoI)gO_QK9K4iG_MiK7TWNCy6LDis0fBUT{*-|rIuOp>futh_g9&5}cd5GzDk z1Ym|0fmP+$d53Yu`!XBhiP}KT)JN7KqCGPG`;Yzqd=v0zRe$p8ZzJf>!SJWx{9T>= zIT-#N41W%WKd02@KW&CT?W@0AmH&qg2GLCjDPlpZh5!_ff+jXsp(+7*$J>ND00s!a z4iyA{BTs+TMfa&63(I|hG~>E9JDS$48cV?WJ4d_-nW8-HGaxZEAC#G}U?`vagKaBlGr#2Vl3kM}S@ z%;=-Ant(xiF2d{~0g%3p+PQ6n=ZpI9a&yUInVw`9F>Ne%pZ3);&3&#e$G&1VS!~|3 z|GIu4|GW{#EfM#e0AP`bV`+|^U7^!#0&t>B0Jc1iQAMR3C&!pJY%gRTW4DCmIN>nY zFk&VEP0gsaR06OFBLLMa1Ry8`F$^I9o!}^P?8gOsAOO9nR>%^EQ4>OZ2E?(>_(CzG zcy=BF@PYsk0DdD}uOe1T1)oa*@LH&|l&B4XHoQ0fE&;f6if1`Nk>M4`UTC+XerpY6 zcFyP!N&xmi2x{U0VQc}}-`f}2 z_=HqMIO3-aDiCki2s)ofnsmE`zcuAIcKp(b!k%GG@y3T(@qHg^Y*a~3VFcp$hWy40 z{EwfA<>`5jed@wlRPqjD!8=~wZFFPv>@gUo!GhxDkH19#HiR}(O9?>NHd<<0yqf@g zwXS40YH%E<+bCZcO?!SxW8fPSWEOSbM)osNlWa1lAE7r39A$qKwp+|40HQ!A^$p^l zDpBblN;fmBkfCDN0+o$*s~f;-=dJ7B{^))ys7XxhNDG-^Qyiwg#a*?R;#8w`Rjf6J z&RqUi38{m5{pJ{|N=#eYRe5;Bc~fj$V^9cw$R@K&Qe=RBQqsc59$t_cNH)Y=X~cyk z9+uLZK?I^oaR({4VhDiJ?p4L~-dO@rESvED$B64aIROWfeeB}i<-688ERM5R7srQr zL$U4<-+URqmU$iK3*|RIVmXfZ<-UgT*3vH#T?^WTSigXe*KIa9M3-82ve-W>$JPtl z^@GZWWFS-e(wCtU`#i3myFoO)WpF7THqzPdp>Qh%jk;DsJ6wtTC!eEl0Ep0v8{rOUt?CF zH8Jb%HDKnV{c)mrJOL1DGpOvx&Ka&(envr>RHk8~6KhT&xO6&%TAx9z(9Wc@hu2hXXKTUw+>>hXE*iqYx)JsCx!QQ%18@D);-&&<}Q|sf^sY{(*rD?LN=Iz}7uS<{7r`%aXoLLfo$GLkYuS9E_2^zzBk9J({|g@1CEIplE^ z6Y~fxwtt;eW^6LpqvuDz^h#}%`0R@@YRc{i_Vvp}-5{*DHR^B&aquGECD(>y_wXwL z7zLGV|1)js-SxJl!lvmU0$|%O$WuFPCrTlBd-BzuR5L8>U68|tg-FhMcQm*2v+;-X zezn{MlZo9c{k6(cfTq^S;gpvUV+I2N^|~Fl;&jI9kLcs>uvwTGtENwUnU>A$s1*FGN0jJ{=ud zE$dbo)X-B25!hlLaE_CEz$yhT`&^LimmNK!9ej13-zbEG%IkvlF0rPn*m9F@U~Z0$ z@R;?c_nV!GU%|edzaD?MT4s600SvLQk1=pGb1>GK)gX@4KGl3p%Ux>_8#=guqu+Ee z;S@bEC1Fkh)VXb)3u?N)ts0%!p@;oCn0kBa_*F`@Hx3dWPsgR^hfB)K9 zS77uwuv|Pd%9V|QX%8Q|4wT{e&NPOGio~>BdPT;J>i4ekhC8;=t?5#SNRO`2bum60ZW-NR_*bF71*s$DRe znqE9{sl?QBwR;)tRKD2BoMI~5v%7r=xB00LMnJqn2IiQg#<=!e%u2;f56C{5hV~`T zxCYM-U_}f&kEZoUMQ@d@l)<`qD-(pnSS~PDlUdVP#sj3h+)jr%hnx4Fjz~#oA%19Q z$rMPD$wZvbPx2F?cm1j?miCE;>jyET*E}Wp{c9tx0ayd$ok}^3uIOjOCXLvq!-qHO z%LWVQ>R~MIa}h*8pLY7}D~ZM2i$1O-iW=FaFLU-7SH(+ul^OZq4kMUd_0hKtyiz$` z5#l9l$c4J7<`lY}!XW&lC3RSVsH#aHDL*Wex~S{scjw5N-ugJUUykILD!-60w^#2d zh6o&pj1)*o=IN~zd|S;`G3L_~E+$fnD;T)I#k!xgQZ5t(*@;+loQ^PUI_(YGSWq5- zd7BJp)do4tYK{Ng_kZuM^)Ot_jj2$3xP09c5)8TQWiB3CmInqN*LCH8()Y9-a`@4S+488nNtW8(H;JNy9tGc~wnB7j4@cX(pM@_BYH6!19Q)Wv zR@82fw_&Y!T0tw$esxd}m!VZ&)t`SDmK+RcGA-468?X54B|zQD7+rP5#!oZa*l@i* zX^O?0>_t`5-1O%AHoCa{fyPhn-_Bt>Qs-DbNBeYxg3F>VyV*V8Is96li9AZ1QDr9A zT~-feZ0NV0&3v2bLLT!fHU5KxMBdR9??BzX{FUPDkg~#ZYXU%J`})#wLIw+`;Wlhubob^wZ8FkZC|)saffxp1wi;_8b($}+4$2;?5Une609eV`>60o^-~C;_F2$m}-L#X=cV+2?02^0_}A%S`B$O-Y?)0_*;243|a;w zkW-950Z_;?_@;}(dVpZ5(PhNS5NNw;y!5$|tnR-7x3Q%ebU%BMV4oSwH^0c2Po>~7 zbG3~w;zo6jjd+N4_JqNNB3qoJ_qQI&H-q<@?g5@zkJ!( zreFBA_uAW8vI<03P%)t&DNs?-scjHzt>Um+Ez8=w<*>d)S$^uKGbcL{&-}O`FzBqwu)<= zFW%PPXO!TT*i?Ma7MFNePC!E{C+3M#1e5P=*^ckqEh_qzVVp^b%Wk&!dBp}q2B}3~ z+|^iQongNhrLI`~Z4btrk}BFM-HU!TxSg(?`skTZPiJ2Br&)`I)Af)7`59-kx3 z5kEB3?Z$A$#;E(_N0~j|4+gy?y=8LQ7Uw%^MpOdz%)ifxo;ZGTOrUh%kk_)6a%sb- zrJiX;iQ}x_Y~!4ZE2W3T8eS$@%?jU-kZProjSqiErM)9h~}i-%fL< zb;yoqlnwHc{jtnT%nNy{8MsS2Hu8F*GG%KXT2(LBk{QTd$#v-mqU+H^4mWJ3zgo}H ziI33|yQP7hJ>8IxnQ&%DOl`)irfW0dQIBp_9KSc@FT2|IAmhVC>ug2QS~eT`K|b^x z=ns>ChS6{vKGNfR-~Q|-x+M09C9h+qUVkU3$DXy}*ID*2fArUx0~8Vk*^}(@3@%91 zF^;|w5~w-aRVd!X3IR|Pj5JhY8C^%n1wlk1%?SA{#{VdcW&Xu?( zXJn4| z%@tbR>9U*aww16=*-fDEQs&I@^4a$-?l8?! zQrwijz@}i`I(Pm4yLLZ|XLHg`_Uf{{?4P1~stp{QU2EV+6~zxq8c8c-N^YVfxJnXu zbJ?hQ%avg;d&_y4SqffdAD4Q4Qkb&{H)pv8+{IWH@)nhojh=4eEj8t7)rr7DD#(SG zHUsVc5HJ%HHjK{7M6~6n9_~W$5gxU&q^2;)?g46xdI@GxT%L+KH998S7wKaW(U8A( zaefX1oo&a0qIcoBe9pvL=eB@~YrY}W+KwwtA%;aUu8|^Mpmsh&j|z+JPWcf^+`f=6 zL+7UVwBPGirk=HrVd|`P%)DaICQWKdd$cmmr-ZIqp7+LG)XCDaTn{+j`cU|{5F4rr zFBniXN$+j zlbx*qiyEJZtZ_w4OPlMYFF755q?f6SQx zqCy9?XK^e^vaRU>Y3ztX#%!?^90`mwZh*#@$|4q{Vp1qndw%I}X76#K)i?-@cY zkAQI1IWY4lqW|~n|Er$|h9L9nAXHljPC)6I-x>sW5~Ai&RMS=u4&Qks#CoLYAMGs= z`;ou;?w6lW%zukA|BD~_`L9;nwzrYGFJFb;Hxs6by%)PDJWon?Tn|%t`*pD9z8=J{ zfwmk3xJ!A0UJMI1Ov7@6Mnt|v% zgCTm23?5on9%i8grKx2aFbuPB71{Fs<;3Pxvr22d?x+^seAj#KAE<6r=|u~y2dL94 zzB=a%02jCNU&9Z&_&p_+xXQ~aBTNFxU+^h}0-Eag!8CpLVQE@}up$k4+Q7fr1pnFZ zK4Kb1#i^!YXDB4cUyk0(ryzi#hS;;i51oL3W9l}ZV+Y#yfdA;;hCRadFypUB9H^_Z zAt&OXyTmaDnZ`n9QX`$TH!Oe6;kVZn|9bqliXbHj6b8`+vuf z|A_!BOyhD8pyot?o|B)CME~zC;#h~`BnUtU>P#Kqfd_pn5ct75rlU+{+8B-zx_>=F zMp9252|z63C=C~j194{j1U>+RoK=j`7`O8O?-P$NZ=7vz+8FE-u|ads(T3p-9VRBma5K7V zOF3%~h;N6;B!88^$TPB-r1#O_gY8*wJJ0x<;=^vM->Q(>RK$D35j#96%p|)U&)`P- z;A{BibrzEVdQtZ8CQ%!hOS^S@1Yp3ckkWMN4y=A`=P;dL;%=ukd^RpB4_KZD@B=64@qP`E94n96rvoI|W?n;Eq=v zS8(vSSpKiLhW(z<2CW=~W0S40Rh6Zi9gur8&HG!b3Q5wRT<+Np!AzM++nN32Mfed^Cz5XG)Q#d>+dzgKD~8sqchJ~HE_jGlW#Khy2z zlTV8sJumk>mXZImOR?V#6qPH6wXVMjf2!Ib9;pz~HxnNCjC>~L&~UKn0X(?!vfgX@ zy|P{GhD+@^#y$6TUL^EiQR^FNM{T{|<`88UZGSUGuKw*l0Nj!2jD9;vyD2VLGGLJc zv-)W9>1Kj5k2}-uC$nPi6ewio)|5*I0g$OcVQ8kYwo(7c1KA7q=`SahUH0|zkF%<(3#}iCjWltT~Z0!EAlHdFJ3q_gc zzZeGp-f3bbP_lI(>vISk^P$VP)rvhm^k&@6=**Z$mt)Q^TeK>%H^kfcUThsf%&#IQ zIDU?F|J}&5{vu|T9xD5vt=|7TKA$ckp8&KC5P&MkM`Y69nL8-^x?i5w7ewrU|5a(( z)ec_VX-yf;%b2$5YmMTbHnt)Pp*1VsH-EaDS{vK;#O@tdl{fTYr~%)G8bB^Z@VqTd zI%lD1&=@DrKOgSB!EQYz1_t-U4M8J`A__-8yv%-WaAw$t9V<{)4Hce(O^TXD-KRoX zzRo=)OJi%~3|>R;hn|faeUXmci#>~xHj&IVYe<^RUy8ihuVecmJMp?iXAY$v7we+z z!mAXU#84?^_^cp}_p@D|m?`=QomW7E!SH#aYtF{gS?-8nTLCHFLbrX}M3Nda=G%$3 zuU{~JQA*N#Cs{#UA*#hwB(oygUaH#!PJtY{tChO)Js+rIMc`j_a0xA+3&d&_Qp`f3 z?caVP{hpJUt5Rs=T*sTalVrHLPi8{cX>O8s=>2DK0d8^v!4jkVYS1B#xJsmQUlMBM;n*%Z7r zm9NnunbF-=L!UCbx3kl&SC{@9=m|sjv9~y>AlNliVR;4b4W{6kB;nA7wqL zZL}mwq!D;U&lmdgZOr*M($=R&vA9&bBQq=H5w#%e%oJIZt#`@zS4xr(8S$EoKs4SU-3da2N)Zy_ljSh>Z{61~T z+ohgxHKG-MSJ~10I@PR}oR>-idhI&)oFca2nR#6NfaY70%L)(amXE+-R-NE3v~@in zM*qOHi_Tx3ZTN@6-W+dF3$K7XE8r3OBWYrVOKJCZHjCA%d2g8ckkMwS$(e|-UI(S! z&1>$S$<~s^{?{h`lep?r55E}|&QQY&*A)l=wE1X_e|ghnt*wQ=K_CW=DEnoAK`G0p z(DSiD#~2*$tWjIAUh$*4V*?GR`110zoE(`xH zoyyyKTl}3}r^6W_CRhQdLf%Lyu|-8r+9qAj#_5=RS!P_wW$baw zEj!B<5|~SZDd%(BCsbD5G=rLGd_>$3U92%r2x4f7kU3K?ADka9JRs+(sICeXP4e{B zSJDXXy8o!UP0}7+ZAXSMCIH56EcmOCVh}qP#RcI*aLCdP$m@No%`-goURP`PWZh5K zyz(+1gn!}*)6{99P>ayMjDLC^^PqgiaM}*ryZ$anQ0DHX?ISO7(xM}X<)`L%>`xz4 zl1ee|NSv?j4RGmOb!ERhkUy(p@SsiCZ4++77)fV!JvyEtz2MmK?e__5L4&%&6U0X- zbbSHq=&LMgN6*P1A+V}Vbae3aNWj`z8+<%Teg5QZh z#A>9 z16{mi#ji6&NOZgG#SJnIvmXJB^Bg_SaV1INDf*U*#R zo&nUXuAA#KR9%0jiXQUGk|0I%%g9TQy|bDCmaU)PeDpJ3Rg=26pOPtm0CTk*DJ)88 z9z_afsw3%Nu)U*t=hRlt^Q}s{%L$Zr6HIbT4dUJAY{JPeCVkFy%HqS<9z?n$$5$rh zSo>-^XhWF{)E<(h709(}H_(+1ap9*c_HQ#mT(+-P@vo52RboYsXsmW*60gdcjNRqu z7h?ayK17tOK4~Ct3KMMJEMbhRt+?==4N51&eyKuB0`Ai?#Juv(t)C4)8i^5|N z_#2f@-Ot8fma5cBc&)qkjGcQ=MMbS)$b)p_Rr7CcQ&yEv+s~GLf5IKF-S}OijS0s; zA&@^ixOu&z@*(S;FOM~TT?A4>Z+478cNQZLTn zEH)J=jw}%(7PV_K+?z zDx8^gq%c=!RjWIFs>??Pcxih8Lsu~6`a7o@-~Mb!c?sdH#Ytt0*A!|c|W=|=-oHiKhf zY+md>mBhCxwn6xzdmXQ(GmLHHs@vr}y>7;6qa|P87m@BQG7O z#_~rEi`B|3R?n0@iCdnrvhLZV9>iuybQI1rXIq=6OPp@+5R5lmW$31zKAgtl6${OC&^mR`#KAK`vu&YR( z5lGST#ln6Rmd$^`xVCsW`?r~=qcKV7sok${mqR5*ODmr*rPm}xT^GkTEl#Eg-#X0> zbh?4Do0H5br>4gEE2)f2zNL!D&Uc{~h`IJL+G|WxIehWx{oNSFpN{-5QnTk`af%~T zsAQwG?O`LenuODyTT7+M(D=sg@3a`PKA!;#&QboLQs3HGIzV&2@cDac(Q0G1QpSE*!JOcs#+I1TdK77c!`P(JP@teFuoN?gr5xNdOxmuMVahrZY2 zkOI4_Py0pT9_S7qyFf!=3GtL1F6GL8nczC^dt{8+m81%3Pr(4)08Sp~1Eva+$-4>Q z*#T>3L}gC13m1O!Hk#cmeUb6%hd z&$LWyqjPC+^%N9R+_9~;%v8);RizFwi_(9&(d4cLv5IRu3eGSyJwH%KPf_Z+v!I#0 zNu~xqUuISW=cqjFlsA3ZT6w|Eb^<*%E{^8X-aG2PM)^HgQmK)tRn5xq5IHg(fx7g_ za2*D^X%rx@*F14_3lD;Pu@X7hr$Yx_5j3r;!sAWqC-jC-f;n$^MtwBpxM>_4NX*4c zVm5;C7jYA*@?o+`xJx?XY#?J)VWgy@>Y_XN^PD#1p6^=Kgxcg6%8S~^`Y0qCN4)7k z0Q7iIQzmtiK)2`%DB8F*9z?}tnue$20rOxGrA<8Cn@?DC?>Ei(U zM^pMd6~-#2Tc;N_j;p;+5bQ?#_zFJ5O*|q<`~;zwDnB9=R;Nkre-PJ@G_rL3vnp^z z(o;6Yw1fYhtawSz$p;6eLdM-2sqlQarsf26Wv~bv^|Q0ba!Rq!29xxD&FuC4I1=_w zgM>q_n_g^OZJN;k77rt4aA|*e+~$ z$N(FQWmvGo*tK591fHo6E(~rveO7Dgvfe06ke=lx$q0;?WKtVpy?ZT}sG2j*4=9@emxS03V=4~n-AV+-73 z#+#0rpNe1U+{~R4jJt7-T>IvBLI)$G+#qjsi@R9#oGM?N`VM=gBP<*+${}9X%TkJh{67c01v|L%i*p-T!z4LE8Bl$Mn3mYdTWe0A1X_{y2 zT8}))d{F*lLT~(W^w0MnF9bRGlB;Gf_A;&-#;c@z4wlr$Q#%gsagpH(ZiwSVbf8JMfS}xbg&NI{Tw>fZrT^Jz zq5hq1wTIeA)x#3Yh1SLq3xHu?HpmZOk((g;hEXglJUy=iMi(9+0yuV}tc?yROTmyU z*c%stzI5>8Ec*azg$az!`L5z=5kJ5bu|DBmFj6;&y-EBi6#Vqg^9i{9gn25SI2!|o zKFE*a!Ij!w5G)yV=gXyw{1E(=O@92_1O$dEbrv)t{^Og_JuD@ez0RQJDxT0KGE&%Z z!*0P_;f?*;t?A$Ch5SY;QJOqR^l$Y3POJL)06H*~{g3o!{3ETh%(PmF-{_6{joW~g z2FUk+*#w)d5telw_!gJWm*^)ociOKFF)KvF(pGnMbecK+?)pl`PQ*?m{ob$t+Yk9K zK<9slCXnXUzO+M{S5`$>_{sCFG+pyow`P_v2+D3Tx4axm-U@6%9iR@@ssFM=|2q)+ z|NXH3ce!v5X2?;rd#Ub?(RluD>zsVA`oqHm=}T68!9#q$-!cFqUsCnoh4Ft=@!W`2 zDM?t-ug|1*Am_9QQCV=3Xtf#sPnY+9mofhZ`sV)+O#mi^nreKaoZTHQUl(;doh_Q< z<@zjn*}s= zLpQ1jz;c%|IH41PaUBg@A_w-w8iD~Jm~X(^O2PB9+mMsJ0mKe~02h9lw}Am$T__%) z#`F0;sNw%uJP0=%qQ7vkDcYZId0z5P=H(Zzyl2D6-zUBQkV$?Xe_b%~9hP{`z+q5% zFtc{hsVn){Gno0^(zdG)0*Oh&b9%^6N1G8l<0)W5SK9MN6y|m^7}V9y^Y6OLxKge0 zr0eGbGBi8L8k&2`B=6%6CXU#;RpJAn=Rt~oHd=PP*398R?O*6 zmjCBV{<)PJ_(IcKp*qO_xexl+mJZ!MZ_dAO4W-FzmJe*?n+fabjE`5dsL0dgwtba7 z%6sweEQTZggb5%3oIYs;;5C@AX)+o%TITf#^!*{=BvOGmy$fqu`RfYhB4-5Y>giOa z`L_vuoBl~%_}+Z!h@BvE9A?Wh6RH`h29+FIc;j#%x%S|XtRNz^*G;gR(0P2l9s!Vh za_F%a6~M7~S06E&hgh*TER+S|t$!=_kgIj3xfr;}CH{wjlZOQ03Ybw<@!YU=Ko1>F zS=u3$lOMho{R3=jNrFjUwPTgb@bWZ$8JoqL-37OxM}y-!4Leic>_7*~YA#`qtRiXZ zaY&|lxDrX@s3%{{2Z8E#k@cceNf$)op>5*yBSfH#(1>Vl6pfW3H@(mtHMLH4wa;wF^4DUoJ$4$<`S(>leZ*deW|ELAf1N ztx4{1-_1l`t+RNfF|lEx&ZeMtLFTPOh4wUb9eZkIqFM6?r9b`#xYrHNi-&tbXS;fd zt!3z{2VOfe+4bL+{s@a9w@1m-4JTG0oo@wtdCfFL@ZaL))MM9lOt8V4SVJ zXU{mUU=Yw*zp-9!1-slzjs?b6CQd-iLX z)p;DlEF$7YdHK#%;XH&IlhWJdGMY!Zv#RS475x^phaGi$LXp98GeNbGNkZb6HYKJCF#r5y=g`vH#?+&pmxXN;S(%ux{u4 z$Mmg%GAR9OqAq2aBe{-97OJAw94O*HvR{@juQm@*n$V*|&f5i~a1~>QXT9y3#KJsh z;ZWVE=ev~D4#haXB^DhMm8$5dpFCPh3!2M!6}~%O7jkQ=jk8;|GExhf(%v}G)#pwW zK-N6N!H=!s1vQ54+XXTrZVIC<9kbO!TMbjAob400lbuE`I4H>1P6apg#IgesnN%u# z^8!Rac6eLY4YVfHcPZbG+qnAN8kT~5)wMn!uqcb=_vAxlN#YmcL=fponG_tGogG$W$| z5EsA&NZE5@)pOfmLO_+2e8JkADeG-l;@)g@7eZj|!4W^UwX{Fj z%!8Q;`w?}6lduXLqgv;uYS(942W8u*kfv`Bq5_S|rh?{FAY9F^kD4?e9A#vn^EdDO zfA#*&qoL4z7RR^$e5hcyXfS`)dh2!pE4RJE7jPxTci^0(Hh(xKcV}3J&&_9CIkB0! zGAKQ8kALuXjNr$wRy_;K^m9z6ZpUzwEO$QC6E)m<Bt(Se~e!m&CYuwb&&*y#;MO6jDJuO-|yYJ|&dMz48z zx4fIDe6i&>OY?Yv)m9_gPncX3%v^7jc9k#1q!XS<@a~z4eWQ(a2RH$6 zu)Ok`%H)KC*w5p$kq}>{6#-b~BLI@ycn?lpX-FYdDD~uXJ*|dL@INQ(@F!!U=Tr*G z-KY&m@?)58ZwM?@h1k-Tna7!(2mBQKLRb0YFvzqiO!6!$LSN`}CZqC@##bd_ zTB^3H?WrkQfm;(sXMUXkFzc*wphw=U9`DrXH`&q8sD5hau5_VvtYwL>wN+@*+O=!) zG~ZDj4e}C9AG69uDzg=&i6xVW;SH+itjPg8cyinuHGS4s%qkq`Z17CGOCR&~)++JW z%B#w}g)xT6d`Ld0Tj^9&`M6eA zF|tyoJ*u|q$}|fNVNR*0_SJP0yu7|+pyIrkaVb!wbEAuuT7Rj`3tH&ri`iG%*$Q#~ z`YWsTv%R-qYnmxf|4)a>6XTVaBzn!RxQ<11#KV-Qz;$ULW`rvlam5V=Q_NsVY6LqiV79cBGKOV zuJ&17cVeL8UDq$`e{I?J`{})|*Vu5gZn&Tv&ZXtYC|Z9*X}(IVw(etppNOLZ%X>yM zI_^!rCP81Qgn({E#C+H#o3_T9a5T(H#YA9Zu|%=lUTl_yUoFqK>MS6@$?#I2-KH}e zOn*4$rYM`PQty~Cd=AYx_}V`4`@}@)UDk8In27C15=Bq9`KO1`)BNzhrEgzy55{yy z>QASprLp;cWU@H3zX7M|za{YgJMYGrrp5~P`vZf}AzM*BD|(GP-W=MsgEuCK7=_VY zEb$B2tcsKiOy}O7CdX-WIPMh6*A9uOjMndRc4xHEigj^>ir$}#HkEiT1$#rY7;@Hq z$Hl&hp6=K4zK8$QHYoqNB?|aCf728HaQX<%fnzSu&_N$#{ZX4`kW*2sb4YMnW!%BM zEil#e8~<5jy%1u&f9G6AyrpR*8T)4?I?<5=xMN%>^dM(*m$jP9W$LA=-@Q*BrYW$i zPs*$MQs%IF-2&PS!dcCn!UKFK$dbowN+;DCzZM)nb?|&{QuTUFV_a@!ZuEZdtw|TP zv43yz*M#ZO7c>J$`#9B288B6P{f*1(y{~Q_%aUsMwASxE3RE(!-@g*;JdrO>X?C@d zeK=eBwSN_8-?iItoE|5FyQD@|tkd-3s(hu2w`g7E%>vSX@>=h`pZw zSZ~ulZ+t@hgNa;k=fWmBJ2pD>^eNmX&Xno7_VDUde=K%DD(S8!pMy+=Vd5t`ymD#N zL23ZI;INcI7RDM2$+bpTd|BfRQZG)S$yk3CvD2J{;}!}VMqDfBhgKr^9HK;z!Py*@r|W^7!t;E^Ff`XRkKVP$ zvkq@BR>UsqNkfugO$B=S5}cW9!Dv>RJUfg?V z>+q1S-guX0^y%(#z&6}V*^a~oK~4YjY2r5;{%DH@xzE3-mq_;eFSD;x`MQ=@kSq3*yLhh*$o0Q9=WztW^)rJlPp{hf+E+_P^3D25DYLjg? z6x6$8zkgrYRr%%yG3qPlhf6;1D_0d)uNhy^nbJ|A9JGb!6*I|+>Aij1F;_o68x>K; zc25PPo0O7g&h}Gdy`oj`<2b!=Z<*}ghhNGx3B1dsN*%8hE>O&XdE*=uYf~Kk&Llfg zfk^E$m9yK+Pll(}=^38u1rJ#Bh>M?N9P2#bBGm~dj!|JU6l{yxO)&RBARDlq+C-ok};sI9IG6=`MQwFUchQ2@R1VV zRbqP%H!TtfwUlC>fsIN!#?f#X`MQAp0#DeJxH@^|IZfF)@it8X`!$cz5;rX?DXtki z;#G8Sh={w8>F#7f1`o2KkaToR@W?pRC1Q+DnIwksZFJ0qme&}4jJcORyihLRDsjup z+}+jnMqy#Hl2et|8!b<_XD@$t^O-K==Db3)WG7&d0?nX%)1KDZP>i$>j%>q}y9NQ~ z8}ao0U+ukjSX0}&E*?ciR78>9l`2)~N{NazSxD~!BAo~kkY1u7AYDK}K}32dQbUhQ z?;RwR(0fU!ffVlC`<%U&YoE2&@44r9&bjBF>mPZZjG37^#~gE%Z+zeTz8_`*r#|bB z^KK9~6`t8uaOs1Zc!i(VyI{*}(@*zs5?@kv*>H4Q#M)e7E#k4?9uA?apQ=^I%4b zO_^b_i0}yOTP79lw3|z~>YVQR8rT@2^pBbb(WcFdEg1Ni*-Do-#}}EjJ1DC(Jq&Dm!4-nYrA;dQ5pvE1Iku17*`Olg*=Ll0_`l{E`u(g#0`V(A#55q-k%mfv z6{QNiwF(fO0r-_IKsM|x2Oq2hh=AWNaEHMhm0-SeKu#o@V>9sDog{z2QbHDxBx-+G z9oP?l2U*#BXiltY9f~Onk9&!gylm}e&w>vBm##$@b>@@_F=*Cf(09Pqe+*KchqoUQ z!yG01-v&Z1Qn9Z#D-c60faZ*Wp-(o13v>gEs0E*oJfvZt$hbKB8D<>oJA#P89Vq&L zgIRos^=6tMe96FHY$W+zI|enJ_A&pJwW<^H53(tCGmQ9()c6@_R~6*eOXYpm5W(G@Y7nqRXzHT7`Af#=ATo)yR+}C&=WCyKUC=7 zGIrSj#;(S?4AtRoI0;Ad+mrqQ|2G=o|MLFuNWa#GRDKK^OeG`(*+HoOg-@)u`)8Ji zz;y4Yzcg8NwTv6doT3=!o?7yMVoLarW#8!va5A@Rrp8+;kxeA@ZqrHtXi-gbl|EE$ zTH}SsB=Hk+v?0VSS3qJEjO_<-FZKX0s#k;X92O*VqFNE?F25aMM;U;329H6k>@$il zCend-US14a$|2nbe~aD_NIhIR21P|<(!V%q8Nz-$Re3+YmjD0@`lZ25cobbztv~|;mjYZWJuW|K=s?1q=9Ncr7;x;M6J#Vo>};_+S8G_gf;$n zF8J5VhW~u9pKt7ESJe7B#(qv7|2=0#DfQZfS$8A$XX76OhxFq){#|@@gI;-Z?3ol- zso>N*SUnPAjr)_tnAZ~a$SO^xy1UtV`^1gC8_N{L1F(?-1j`aJwya9--v}&~N zItX;viD4Ctu=I@~*jGtsQZkJ?pC(GS=dQ_X2et38dC8`fD1CCeAQhX#tx2d*Q^U=o zT7#c%?GlAs^&3)6n3$EbRy&<-toR?PvG#nemyG9~KjqtaD)-F3`!2!QJeeg?_!u-C z4-j>|D`*$U=g40dr429EVI1pjEYpWW3ftM>^}07A8&RZr#urg~?$l>hmTM!WW~nw# zWP9>983(^Dv*S@0=1Mw}Gcga>sB-MA<1YUt!p83dp+%yTw ztkvH0{beTF;J} z@f?GS0T_!^0S!pcRE;E?NdUhq&@%wse$m&#REvCM7sL1p{>5EZDl0Hcg;On0MR>8> z0Fu8|ft_e1QMZ@P_2sCI^SJb4RvA`4I(Id0r821^vM1xrmgE5QTdxe%9mlqdN)(`Z zkAmm($@=3iUh#0(2Vos?%}Fe`<5mVBw|%_>XesSRRk*dAPM;QwiR~a~*<`EeSv8X# z7R<(;>a$BYxUC<;rnt)M`s;nG`s&UX1SE#*JovVro^78x(J?pl za-hO-i=(C@YLy}*33Z>EtG~+1n)B*QmdjDDTG9_vhT;Q6wTG$u%~YqOqeozuD|ogz^doY>NBldjNB=lsS&5buOub@*Y)n6F%D@lTQ+h`HrTTsZNP~h6JhF)%Kns+=QEmP(9R6{N0H0_cxZ}ZkhFO5AAKzumPs=uX!sHm4>DK&~$ zinHm7=6y%reV}IomLylV#FBqQ<%Aad-m;xFy|&dPnHF*M3q4kD;*KSEI%U0c%!jDi zSdZ_1-A4jvVC6;Y@nA{Zs4MIqfGOqg9mV$UR6 z&K$`jcPe|D<#n+az>-#TlO zdU-{X#1f+yXP9urUG})uA?BvzE9yHe+v<P*Q`He@pcm2;933acpO* zXBH-hM`Xm57`Y~JXpJ zOxo46UyH?l!L4MW91p_pP_f1m@Yy`d z%I}Rn1&|}NL|bts&rK01XRuVn<3xQMQ0_!sl)S~-gSBVdlDWn1t+U`)dqT#IYGqiZ zaJRfwx^LS(9hCRzlNL`>P`A7+UA|W`ae{@-GY>B7HB%(hG|~X5%F)F+&N63FK{LDk zs7H6H-+F{wlOyXv5+4%iH->R>Z z_|~A4SW?-oIr4JEfKuN;T)>&((?lXS<@^pGgiv2P{z#<>DV$i_b z?T-itWHz!hDulg`Yn+kSp?4!sG;R(GBWjMBSjl2tNrZ*h#VXp-ns26Zjf7d;! z-G8B^^Xf{o;|l3UpPTg{Q>)1!+(0*U>9gd4bDQwoFHq0V{%)q7vDIXpa(2cA@YSOb z6PU3RMC;M;LDLqwy*t>k9%~a~Z(C`Azkl@hI@|Y!OKl!>E!6skEkH!Ra%5H+*;zKd0ut&9hLq$6T1{_(N`o zMJdxPr+U(5WwtJ4T5{-Y%?nT2YAHIeVdK6~Ww++SqhuvfzKP}*Y9$5Z?>^3*sC$FQ zSbZgTat)9!0=o28quFT$BDJaED$u1V8{f^u>D7mgSv?lj`z1ou`MW0TQ((s}e+>)p zie%ZMjt*Y$rd-yeQ*u-~5_op`U4F?_W5AaeK5rDQJTvB`0_&J(sV>?bRU&i7CfCL! zZA{v<9&88}dq~SXip^0F&P_26#5 z>D>yzegWKlya%lDv@rz`SMG%LBrZu*Pnn&z*kdT&rC90H_u;`ei$^Xz9C576FZm+N zz0{(ve7Db6m8W`U+{e`8UQ#!2N$F5zb9u=`TOLbHPhWl5yR)X9#GZBh+@e~xV=U0<463MQO662oyIyui@t@+@-w!TbR zMOhT(^KLBfH&xAyfIm_6)G5*(xi#uw%-~ zS+9KacCVg=1G>$P7_*y^!r)3vA6{7UdD^CMhBq^+_)i$mISJvkx$i~3ZZpwT3Qmd7 z))J%=+hHto4uWh-AQ0=qC92U6&}g8wopQKY5++xiJ2m>mYxza*`xcV(D)JHc>)6y&)TM(cG$MM&s;*IQARsRut@oA#w8R&g{<`C2$TnfVHTO?L8Z!Egnl+DOvNZe;KwJHst-HA;6n zp!MSoM`3HX1zhIn)H^pw1$xqlIARM79z+yDyn&yMQU2;cNa#7QKxL^Czt6>!<8>+H zSj3099Es=|aG0@Eea^(BgQ)ScqT5>i$uZ&LA-{Y(ThF&wA6I*9cb;ODQr`7#@KMz6gBCVc7S)$A6-L)%Rj7J?#3Q z;gwE=&S+4=zKP9*o3czK$DFGh=q3!zCu+QxM#mh+jfZaO7Swk;F zh#;n3QlOh~Iy?pqEy?<=ySGcQpIUQo;2V;FQLa^bKUfU_Z=8p0+$?TvXIeLUVx1%x zrOBT%K02@PGO)HXJryc~vv7k}w?o*-Q+2U-h7<|lu>)gQ`=t2D=De8VYn`+=+lm`Z ztjBK|oMJnD<-g5X#sFZO>rm&t%NB~ELbVp7qGB(r#-lLbdK1j1sHj2G9|``5}){{~8k02JRhR z3Sy3&=%sy+EG2MeE*sRSPY+LHPnVbV3Ke+$((RkmYt*ss@-^gKbIbbWpr>wh4HU^{ zBZNnZ2Q~4Vw-f=Y`H2W@*klp^lZ4#^4`hM;3eu5c-x28p#ld!DVXbNftMtYdVdDMVj0>){?MV zMN93n+*rs_Ei8BL^m%TgUE;DQH#~UHZNOJHRfLpwY-Aze6Zxa(sxOTdPHZu?jQy{yuQh4XM;!~YvsSuC~$o`fZ`O}Mf*f1z4$?+ z@_qLmi4Py$N@BS!$u_Je-m_mvbyc&>yxvmJ-1RXMNz}s$xL-B9U9^er&6%C-6&~@W zkPMlH2iKSdLY?VSGVO0izVd)Di<%sRGMZxOAK#!5KwAhRQ#Lk7*)mb>9u z2fJ-f-n7_O(dh=UP?PO?Dj z;?EwZj@d;FqQs0||B@(q{YqP&nAc{99*b9o5KTx_@SOHYC4monruLKRxH|1d2f8tH zrA%#Jyb>6v<(Cg*?-xHPp-dHQU@fJ9+oosP#cn{Wxq#jBsm>`6e^@vYAU|F7;;EKM z40kF|J-c$m+7fblzuMI+@2f-&s_wI~#-i#o8o%9-b8XBuHNP|+L zxmqU>)d04eg;Kd293@a%r^h@Zd`+AWA({2ZnoQD2*1U*J7L!vm50; z#<}IHTRYS3R|l(DL@q>Bdxk!#RSiEC@QKlvewtN5eYAm=dE<jha|b8^d)1zY8uiB@Iv}sY%H(?`*%B=?^8myG@u@F<22f<& zwr3;jPp`ntCwMp$?j>t`&X)46Fb*wh$)B9Rj^LNt-ELd5FNEUmZf0T)!o;aAL(F6< z{dAf_bPyp*eU3|mn;D|5+~vW$-`f|;EjSeb^}U8W<3VGo*~@v+HM@Wy7)cQp<9aae zcMR&$090ITDKQHMO;fU1kuY>JOzydyTwt(%z*utkGa0%nueV39HAms2+&L@zHR^@# zmHK^S6sehDYMEW?F^^*zu(xNn_D`m#7PFe*=k(GlU3*`(hO`Vcn6)h0E*-*cGc1f{Cyj) z>LKIV8SQZ|hKp7SHw<#8C}_9h&2}^@h7!1ID}q-n;%y_;8J@*n@2F~-2=1}^3yCxibksy1gzVzrTdxMb>6#u= zK_v-o181%?S}euhsExE*GdqPGsw20~^c3D|YHdPQmbn%gd1jaoX!q|{OJ0`kWk3Wy zp$_(Z<;Ya~eiC!X2A*k`J^Z&9Jq_;WFgEEVog

xh__Wlsl5ohJ35PXHMa8|zXmDXwUX9W-?GuMXp z3=sX|Rht#e44Z*oSC~~fJ_M-UvNnte-4A?Iz05wNceOBhkjR~&^ z$wq;eZQ+THTXvSWI60xGtd_L%&9*g)kr9W;PIGNXNLFR)D=%2r;X(sjqkIbHa6KJM z|N3b*c65JXXo_QC1Z6kAKFZaD;ktBqed+XLGYOe-59~=Jtisotgql>Eji?Z}T;o;a z){tMG5o5B{a1(C~9`dv{JDoj*{RV%_avP^fby8&!W^r~m;YuVGZ z=*R}aRY2P!6s?0PDvI!6x18Khr^CoTaNbGXyOY~~3^GWaDOP*b=N8G{!sJwxzdCiZ zzq|l_&Tyr_eWw15oZf&PO2PR8wj!k~W{XYT&Cs7_d2|$|*plHUUrin=tC;aE-78B!vD-8L4sk06U&z7y!6XP2dtyID zL8D^0aG|WBB6xeidYrqy>50mOH!bHWKZj=StW-VdSM;X4!R|!PmKhO>EATt2_w8lG zg%&H!$x$gbRQAi>mlc$Fk>|R&#Mxx_^J@=8+?Y=dDvrdsa>Xi*^xb!g%oq94 z&M4T;a01InZXs*s8K*ozFv55=ADO+ko`qgoyB**C0aek1m{}XY4sWk|eJM};av+n^ zt5Xq|!fwqcfED|-(__m>$Raf16z|{>2?-Z#m3>?Tf922VnP}R#d%mER?NTtRUvR^) zRc>ISv{3yi1u%RSDpi@I_;@+c4RUZ11;9TfAXfjYiDfRUz zf`0V2tk_s7%!1ESV(?*A7Z`FgwDsMYf!d2BemTow@Lc=-nu-sV9wVQpjDM-MLR3Yl zGDqLbd>mcGRA97b(*Vfd6E?#fE2 z<}a0(O>etnu|0oo&zP@tanaa)w=A+@@nK!DNbxbqJ8=&Yl@47Coio%KUZrm{>tT;t&fxa!Ryb3g5@H;>!*_KVcdm@ zlAkA3AtI?xJ4y_VJR_#dDu*Tz(RtTk&gueM*MlwpN&?sfZ9b|au{oY-P4q*Fs5@Gx zP3xnCOLB0c>}S<7%{9C}d6ua==W)lK9@;S8zZEp&#%!h@RJXsj$XNoqk%gstZJZgnPU1H5?Ffk&Kko62HcrPpP>BKdl|((AXwauLj6nWGh2 zfGnM}w(!XWCIQt52g1CN!7X_!3_JjU3?isP>Qsw7x8)vC(X+^2Y-Smo z0o>g~nid03$P0q3B~G^!)7ywz3h?JbuaP+VC0KQoh9rn)gTL4nT6;ZLPT1Gq>3Tq6 z=X5C-f5-d{e}FH-#yl%GFS;7?Tpz!V1frH|_p-Kl**1?sz!b0V`*p^TH5=#XIBiGZ zEHd^6FneNo^^$?mTXA?+^$Kjd#Uk@!IrJ?&qTx&9eB^-qqUtb=V| zN+wk+6Q&CF0N?5SRVkuN^kKc>r;`P%=Xz~9G!3y|7&rNwX-@R|B|s17c!}F~ctP|| ziw1svR6$YZ^wh?=$}LPfAq{FT3%%pH4~n} zH$ME5wb`K`$TRnAGL8T5OeJ2aty_Q=Muso5EgWWf400r<69xTy_u*wgBIW2>_txH{ zI!o3gtO**E3LA_jDFG~9NHY?OKIE*CB8H?Lg9d!Uwgmc%9$fyWou5$Zg6tR}^w>Ro zSU$V2Pke|VUD-MU2yAU=VkG+ySBOooo6B5-%>V{S4m%+dFcbQLq<9BJX<)&El3J?y z`N#mu+bcZj18f`EUs);>c+tCn;86?`&orO=>EfSX_r+~AP6N5pdJN)(?Tf>{0lVaf z;6otvP*VIL4D|26CDk1ase)MH>?RU7ziv3njId3R52##=!B7#9uyHUksb=+06PEn5 zHp2hZ@}HVA*?$<)I(YwkffTxOl9!*6|gaNrMn|o2RU0(iLcM(pvdPvm_ zt=`B&&u~@Dx^sK}s=K$93v2FZBE==k<@1;09w|o^FZTQMdYT(Jis>|z1h;H6S`8Lj zSAXw$f;y`x-7|QoZ+oe=3lDGQ ztR(fM7AbQj`R!E@oRJ9|5YaZ;&ld-DKd@YK-V|usMW0{wrz^$5+zOWsyc1G#h_&6H zD_%{?B|pG_;*sca=k089AJxKeTi(D8s0cFuf`DD>(H@T1ZN*oWp4Gl{YZb`isrT$n z(5;7}_4PtnlkL%^gD!{mb{+(fKLEWecQ1?&&bRub?Ry&y&|^9eh=AzlSV3{I@)z(n z@$ZsJ`wnZlzw%}O#OVAV|NCF@g#VTl^v4f*H~T;LVc6Wk*HoOD8D&`?tP)%z-|L@< zw!@2{q#UE}k0ZP-i8ihuCvO#P;qJgzfDyp~}it08x&?1JYe!GR_62Npeyua+~THWNqd=;G%0_0!Xw`5y7x|c2Y<550}(E zu>{0E{o`kw5lnBzR2(}QQ0Ma=Tla|VD0Mu0Cf(~$@d*ceh-pZqOUW`;wzj2nZ)Wq3 zi&d44)zJ2mm;&@r$pF6#`j8w@bI%}MVn6Z+CU!6}-hP9(v)#jKl48MNvGOnLXY}?j ztJkh&3MFuWcY%)g28QPc&zu03hb;mS*c#I$|EuS|L;eD`fd3m&dwqCRU+Lv1m<13~ zM(z+MkzWxEz14d~-ajYjKig;i>*PGfrJ`9|njTfydNPgFR{@aM-odt}0Zw7lKMSb! zk@vw+bjY0@Dk6b>PI*5FP9QV_Y(jF}XzyFi|BN5s{j<~>2u@4`aG&5C01NUfNgQTp z3^261zy}yuq+$9C0RU+5+X439>Z;H8=^P*=@P~CD4&ytC{NY*pf4qalldzw+{MoQS zyU))-@^dQq?>EQtN0o^u>|f}=71|Y(`}hP0y{31-?JIY#{J1XsunZW<01<`dt6(CQ z9?;=bT0f^hN~0atr>31DaF;O40$M>bP!p%y)Bo*s|j>sKXHu+Gm985RF=3 z;{FFDW#0WPwqDFJNbLXL6*K;E!Mnsf{P&ke%IpuTY+Gu1el zBnAiu(>J4!m|@$=wm=Sfz`3vQoZ8J^|2NCkL2LaOM70A}MgHOIfOn_jo{EGccO{QO z3zERbHUjW{PzGT=NWo}AHqa8-B=GPD+a2XCz=B7;^#L~$dIz9?{Qe+2wj}2Bvs*+; z{y``;81w)CumY<%izV2o1_G<*_H zb5!o43Z+h9p3I0SP`S5xl`MM}B%F~nmi>7m(>p@S_1ALWDa@VdkMb;5W#8^AA@*vB(^IWHt_lS<)XajJFg*hZXbmKU-)I!K#fI3mkyn`_$bIHNASs{RlwPE?w0*@k4QC*;T8(Tk^P=XzBb zHl7ilJE>aiC{Idq8?CS0;TP^9BC~KGbW^YRYGnC$!7f3e5^qyy=>rPg1eiC}9WC!1 zBw*>omfF86Qn&Q|a^>`$w-<*8edVw%MOaalI&15^dgKcsA*-knzQ+-j)qD)G%Voma zr>(7DUp2q6(9WkL+fa{}G_ya_Xd-fc^Nt8n!y-a5%j10&3eC*B*|Ir3E#GzDN>E_F z-&T|$SBz$;DCe8B^x7}gmjullCtA$Ggj#& z^=pKN3vIAGAEziQ+ZV*RdbtLIv`cTt;wWD@)5T+=oBglg3^1{sDsf8nSm;Yfp`v+2 zU3CI8^R!@h;WDq``#JphFd-ZNf|fayb^)h6S7hAV!c;_@3Ur(`RWKQJBk~VW@Fet1 zZd`lemt^#d!)9S$&0*S!rsTSk$hT`s{AHT{tM177k%HO^bZ5>Z`<-o`4Ih8GRBIoR zhwvP?>C~{s<|$qzW@9S|2;QE9=<7S1cS4G|OU^K|_McOO%9fOLIlf_tX?gZgpcYP# zxq{1{+kY_%_nzaGh)aXS`n?dv*Mm)1B>GtQXSiDr|84eq;`(Zs)N#`_r zF>&UxLd8ZMaz(gi@DSi;{q&=Y>kq%r$%{JFw1i@iO&$&GkoghIifbr!$<__T&6`(! z+or#hEcv(tHlA+abF$ps;%B|~YW=8ZH7<@BI^6yie*?9e2(^duc1fLyzZM_2Fd4w< zzWLk`e=B$>OcD1wC=T_o{-bZu{ zD3|x-pM5s|%|*_n@pki6efl`sY&ns{Ql02dl+y9>_$YXj%o&kj*mLKCWc75!^gI<` z70!Cr&t0vJc@#NpBdY*OxAe)_NN`fSoryxgT-(*dO(Iyz+5-(7O3+24!#RyHX+8cMEl+m)9nkM6B2pU?^OIYFsOJ<*UE+$ex`)lf>j! zl3Tb>y9n}mV2{-uO$<_)>cCltpoFy;@P(zXczhHC9C%VHRfA4WY;%5@{!iPv_`@4! zrP$pjrXPRs(Y+0AL{sCSY8dwKmID>%(NxW@v`q%|rQ@xl`Oe8UN_Bm+{sB%~ zFvaY{MW@hqTNgX`B9!-Yxz*d&GhR%G=(n5@ctgW-0l&fCT!QNuZE`nu3U?M*xz;i} zFsGNq9i<^&%{M!SfIp4b*)VCif*J0KmdwdrHBj{^SdA@2P@anq=GHwIqeQ{$@N#Io z`Q5aw-AHWw-&KCcF_n`S9ZEyWDw73_N0BDuOUU@BurR*jjGjLn*L=Y8|8G8pU>k>kdNM(JMOk$=eiLcd5Q1&6|%r(u-6 zere4#OnO5}inVGvt(PaM?=*{1r!Cxje0?Qs{rWP;UvPH*&gIc}2aH3GExaQYDbl$A z2hrd16Z(L3{sm?^_a&_^x-}|P)3!j#sZ1@wh_rXMTyLs?+-+1*J0UJRs;<{R4x1{cFsnSM;+mE|#iqd-I8Lpa zIa!_1S$j$r~iZxRE9GqrUV=bS;4UxL)uwpP^F4X>pZ@ZZ5wIl1>Y1=8d zzJDY{DBm>Hz&>_SmHv%OCI)gZc8vTPU}nA<0`F~!kb+npN4Qf7!9Wxc?giWBTXXtd zn@jgbvTrN+jmYWjnUXAirbg}UZICe9=g*2OnkkZcfU$xWFgS^$&#j`K z;~yv9eAK<@#dp@swc6j}%ci0qkEB~(i)eZ=fE?Nl1zdwNZ0Zaxdbe!WQxdb5PN6%5 zkaautywf?ap|U13Mf93vr-&`blJV6K~`Y}u9%P} zfg*^ze#zLQ@f9&I1E#=L_cy3w!&hf)-rZ+KE`8i?e4|XxY7pelVRu_O#Gg*V*1*^0)wvau7|*4<(TyMZFbnWdU-`Js*)t=Bd#ol3K5NYH z=GB8mOHu7y+I(Lp7akKD-7i8CvV3*T!tlH9xLBkC;&A=B5{)%80&=*D*qtmml2|3o zvIqL#eD9Xvch!-zznE@NGvR1|YFH@q>BfbDwhYQB1{RiLiU~2N-f59&31%N7$LE)y zITa3SA`nqWW5wK+&(3wc@oKOX=@Pw%@s#G6OUK$aB|AdC$cqkc$=nIlSY500eLo*`6T*#Ink-qMV#$?(K60T%ABz{)) zDp+OXj8+NjRUdZb4)#nEbuVmRyrx!1+j;Z+SAn%&6yYjXxW2$_e8u>YvzKuM>bX0j zrXuN~%{{SmTb_gi&CWQc>DU`*?iBAd`g3Cv1AIAWSVc(G$)?8pqb>TV;k7Yi)3X)` zUDSi}7|vVFcRB53r^zBlMuQ1Z3~97EX06}h=wXWc6kqj_giMivQo{bM&t33)=)UZ? zwTlXpbNK2LU)xcqZ8xwt7VV|#>&LvFPmLeoL)KTsD+mS$HN@7$ZcLyS0{7V0f_vkG zi5q!|-))nE=kX^$J!xX}@Os~IC?hrP)Iw>wGU|+L z>Vf2Nhq&43COe62to3H-Z1<&=8mtkxbQ&{j;@~3C7R%toYhq2_d8;$@Ly6*ARm}6X z*@*$?{)W8e`bqIU?}l9eo}iSeDTt|caeZO=nxD)^PDL7nO0T8$6dEn_z|>E_(n2jE zVC*h(lWGxSQ&N$NJ2F2Y-x2R)7S~ebxV6Q8f5-Cyky9zAH!(;n7Uo$ohaU4t-B|$> z@lXta-u*8tIF)(>baj)&Z}gdQR5-!@MiBo{NO_W&xlU|K3`{ZdMg<>TUwa_GNieXB3jna`xFn)2 zUKTcC-Vt5aN^B++j2?rG)?mFA2W8QCe)g{3f5$ZdEYH7O;xnluWJhewt1p9yMJ8cR*N`j0lU{%d7K`Bz7Tl?V% zzp5bBGjGi^k+f=fdDsz!DGYrR2%W-gfgsCW0XTk9gmmqNaKA3HYzYXa#*Y$r{uYl_ zvlT3GmMHICEO#g_eSA_6aq-UFEuBHs3t=roHsN8oppIJU;FL79Lan!?+40H=1fzysS-n`$RcfUO9*DFrxde;9&RY#s5~EEUC85A{Dn5Uw5} zh*V4mMBLyp=#VZNr?3Y-Vw2dyV{?u{+d^PV4U&H=kckGsBP;fbEWknzQ`Bi2Jp<6j z7_)GnVt6RD;`Q@oH zA#4jV)SF5QFfc#p0%FOo38nzU`o|`U zUI%=3%iO@G*cTvFHEEtk@uz$!EovGW1Sj?9*R;Q&(4V+P7O|kA;ZXgXU-mC{F#pyu z_D9aL{|Am`Wx`quQ}?i7QlxjSZh16rFOJjzOm6+~JM<2WB3Hv^1w7*KZ_j6X9lkpe z>%6k1j>Kzx&%v`N?iYo0?39(cOZUj=iv;;bj#u?^1*COluZ+*HL zmZxwGx>^7~ItT&Qj)dZ2KG?hj@FLBC@b$8G-x>A@@cd&8m>A3j06P5^+$1_Z8%au3 z0M7FJ6Ahx&l!1<^8!|+V0LDL$NW-d)?kciEj7u|r?KiM6;>#`YuEMC!1M^J)uuX&Hi$%UB(ywX{fagXuj&Q&-qf~F4q!+*& z(=((f*vbl;A2#JA$pK3$JQVQ7L55ND64xqff{&(dZOj{`y>U{#G=Eupls6lwQX9|; zux-i-v{Ua53<;1o)&kCO8$E`4$RubOoj^()x(e-n}ZXQE609dep?6KmjF50R!nRc$=vPc;TL~-RpKc73_z56+lkv1OTn@953TVZEyPl zp2DGrdXLH%dl@&Hw>0(QhAaS2;Us?vbhw+mq~tAuyh3vtJJ&V3eI28R2}e<>F2WmT zdO~c(sGvs|-Ah&0V%jD@I)|kZ)8@>NKz6)nyo>XZUsUacT)ni5glBxMjJWgRZ0-!C11)X3Ug*P*Rqt=YD!be02imo-x5bxC?UpPtr>{2QR%VoSV23vyRv z8H|$!`th4(QXdvhJby$*q5`If`Pa>lX17sx9738`p;)z0FXs z@IhoWtjg)9Fep#OC8~q96Jm}*Y^AW#rehG`)*b2>#L7BJapvi>eZn{-HG0?#2-PTv z3vX~xt7zs~#!BnN#A?hv{9O~fH8~hoR0Ow7$@qAVFjz8ci!*?9E7W?1U`r!_$$HWR zNc+*h Date: Mon, 25 Feb 2019 11:30:14 +0800 Subject: [PATCH 2/9] Unit test for nnimanager (#770) Unit test for nnimanager --- src/nni_manager/core/test/mockedDatastore.ts | 18 ++ src/nni_manager/core/test/nnimanager.test.ts | 178 +++++++++++++++++-- src/nni_manager/package.json | 2 +- 3 files changed, 178 insertions(+), 20 deletions(-) diff --git a/src/nni_manager/core/test/mockedDatastore.ts b/src/nni_manager/core/test/mockedDatastore.ts index c5c35a33b0..d08b5b801b 100644 --- a/src/nni_manager/core/test/mockedDatastore.ts +++ b/src/nni_manager/core/test/mockedDatastore.ts @@ -99,7 +99,25 @@ class MockedDataStore implements DataStore { private dbTrialJobs: SimpleDb = new SimpleDb('trial_jobs', './trial_jobs.json'); private dbMetrics: SimpleDb = new SimpleDb('metrics', './metrics.json'); + trailJob1 = { + event: 'ADD_CUSTOMIZED', + timestamp: Date.now(), + trialJobId: "4321", + data: '' + } + + metrics1 = { + timestamp: Date.now(), + trialJobId: '4321', + parameterId: 'param1', + type: 'CUSTOM', + sequence: 21, + data: '' + } + init(): Promise { + this.dbTrialJobs.saveData(this.trailJob1); + this.dbMetrics.saveData(this.metrics1); return Promise.resolve(); } diff --git a/src/nni_manager/core/test/nnimanager.test.ts b/src/nni_manager/core/test/nnimanager.test.ts index 0a2074f2a1..ec98f8116e 100644 --- a/src/nni_manager/core/test/nnimanager.test.ts +++ b/src/nni_manager/core/test/nnimanager.test.ts @@ -19,25 +19,27 @@ 'use strict'; +import * as os from 'os'; import { assert, expect } from 'chai'; import { Container, Scope } from 'typescript-ioc'; import * as component from '../../common/component'; import { Database, DataStore } from '../../common/datastore'; -import { Manager } from '../../common/manager'; +import { Manager, ExperimentProfile} from '../../common/manager'; import { TrainingService } from '../../common/trainingService'; import { cleanupUnitTest, prepareUnitTest } from '../../common/utils'; import { NNIDataStore } from '../nniDataStore'; import { NNIManager } from '../nnimanager'; import { SqlDB } from '../sqlDatabase'; import { MockedTrainingService } from './mockedTrainingService'; +import { MockedDataStore } from './mockedDatastore'; async function initContainer(): Promise { prepareUnitTest(); Container.bind(TrainingService).to(MockedTrainingService).scope(Scope.Singleton); Container.bind(Manager).to(NNIManager).scope(Scope.Singleton); Container.bind(Database).to(SqlDB).scope(Scope.Singleton); - Container.bind(DataStore).to(NNIDataStore).scope(Scope.Singleton); + Container.bind(DataStore).to(MockedDataStore).scope(Scope.Singleton); await component.get(DataStore).init(); } @@ -51,9 +53,9 @@ describe('Unit test for nnimanager', function () { let experimentParams = { authorName: 'zql', experimentName: 'naive_experiment', - trialConcurrency: 2, + trialConcurrency: 3, maxExecDuration: 5, - maxTrialNum: 2, + maxTrialNum: 3, trainingServicePlatform: 'local', searchSpace: '{"x":1}', tuner: { @@ -71,36 +73,74 @@ describe('Unit test for nnimanager', function () { } } + let updateExperimentParams = { + authorName: '', + experimentName: 'another_experiment', + trialConcurrency: 2, + maxExecDuration: 6, + maxTrialNum: 2, + trainingServicePlatform: 'local', + searchSpace: '{"y":2}', + tuner: { + className: 'TPE', + classArgs: { + optimize_mode: 'maximize' + }, + checkpointDir: '', + gpuNum: 0 + }, + assessor: { + className: 'Medianstop', + checkpointDir: '', + gpuNum: 1 + } + } + + let experimentProfile = { + params: updateExperimentParams, + id: 'test', + execDuration: 0, + maxSequenceId: 0, + revision: 0 + } + + before(async () => { await initContainer(); nniManager = component.get(Manager); const expId: string = await nniManager.startExperiment(experimentParams); - assert(expId); - }); + assert.strictEqual(expId, 'unittest'); + }) after(async () => { - await nniManager.stopExperiment(); + await setTimeout(() => {nniManager.stopExperiment()},15000); cleanupUnitTest(); }) - it('test resumeExperiment', () => { - //TODO: add resume experiment unit test + + + it('test addCustomizedTrialJob', () => { + return nniManager.addCustomizedTrialJob('hyperParams').then(() => { + + }).catch((error) => { + assert.fail(error); + }) }) + it('test listTrialJobs', () => { - //FIXME: not implemented - //return nniManager.listTrialJobs().then(function (trialJobDetails) { - // expect(trialJobDetails.length).to.be.equal(2); - //}).catch(function (error) { - // assert.fail(error); - //}) + return nniManager.listTrialJobs().then(function (trialjobdetails) { + expect(trialjobdetails.length).to.be.equal(2); + }).catch((error) => { + assert.fail(error); + }) }) it('test getTrialJob valid', () => { //query a exist id return nniManager.getTrialJob('1234').then(function (trialJobDetail) { expect(trialJobDetail.id).to.be.equal('1234'); - }).catch(function (error) { + }).catch((error) => { assert.fail(error); }) }) @@ -132,7 +172,6 @@ describe('Unit test for nnimanager', function () { }) }) - //TODO: complete ut it('test cancelTrialJobByUser', () => { return nniManager.cancelTrialJobByUser('1234').then(() => { @@ -141,11 +180,112 @@ describe('Unit test for nnimanager', function () { }) }) - it('test addCustomizedTrialJob', () => { - return nniManager.addCustomizedTrialJob('hyperParams').then(() => { + it('test getExperimentProfile', () => { + return nniManager.getExperimentProfile().then((experimentProfile) => { + expect(experimentProfile.id).to.be.equal('unittest'); + expect(experimentProfile.logDir).to.be.equal(os.homedir()+'/nni/experiments/unittest'); }).catch((error) => { assert.fail(error); }) }) + + it('test updateExperimentProfile TRIAL_CONCURRENCY', () => { + return nniManager.updateExperimentProfile(experimentProfile, 'TRIAL_CONCURRENCY').then(() => { + nniManager.getExperimentProfile().then((updateProfile) => { + expect(updateProfile.params.trialConcurrency).to.be.equal(2); + }); + }).catch((error) => { + assert.fail(error); + }) + }) + + it('test updateExperimentProfile MAX_EXEC_DURATION', () => { + return nniManager.updateExperimentProfile(experimentProfile, 'MAX_EXEC_DURATION').then(() => { + nniManager.getExperimentProfile().then((updateProfile) => { + expect(updateProfile.params.maxExecDuration).to.be.equal(6); + }); + }).catch((error) => { + assert.fail(error); + }) + }) + + it('test updateExperimentProfile SEARCH_SPACE', () => { + return nniManager.updateExperimentProfile(experimentProfile, 'SEARCH_SPACE').then(() => { + nniManager.getExperimentProfile().then((updateProfile) => { + expect(updateProfile.params.searchSpace).to.be.equal('{"y":2}'); + }); + }).catch((error) => { + assert.fail(error); + }) + }) + + it('test updateExperimentProfile MAX_TRIAL_NUM', () => { + return nniManager.updateExperimentProfile(experimentProfile, 'MAX_TRIAL_NUM').then(() => { + nniManager.getExperimentProfile().then((updateProfile) => { + expect(updateProfile.params.maxTrialNum).to.be.equal(2); + }); + }).catch((error) => { + assert.fail(error); + }) + }) + + it('test getStatus', () => { + assert.strictEqual(nniManager.getStatus().status,'RUNNING'); + }) + + it('test getMetricData with trialJobId', () => { + //query a exist trialJobId + return nniManager.getMetricData('4321', 'CUSTOM').then((metricData) => { + expect(metricData.length).to.be.equal(1); + expect(metricData[0].trialJobId).to.be.equal('4321'); + expect(metricData[0].parameterId).to.be.equal('param1'); + }).catch((error) => { + assert.fail(error); + }) + }) + + it('test getMetricData with invalid trialJobId', () => { + //query an invalid trialJobId + return nniManager.getMetricData('43210', 'CUSTOM').then((metricData) => { + assert.fail(); + }).catch((error) => { + }) + }) + + it('test getTrialJobStatistics', () => { + // get 3 trial jobs (init, addCustomizedTrialJob, cancelTrialJobByUser) + return nniManager.getTrialJobStatistics().then(function (trialJobStatistics) { + expect(trialJobStatistics.length).to.be.equal(2); + if (trialJobStatistics[0].trialJobStatus === 'WAITING') { + expect(trialJobStatistics[0].trialJobNumber).to.be.equal(2); + expect(trialJobStatistics[1].trialJobNumber).to.be.equal(1); + } + else { + expect(trialJobStatistics[1].trialJobNumber).to.be.equal(2); + expect(trialJobStatistics[0].trialJobNumber).to.be.equal(1); + } + }).catch((error) => { + assert.fail(error); + }) + }) + + it('test addCustomizedTrialJob reach maxTrialNum', () => { + // test currSubmittedTrialNum reach maxTrialNum + return nniManager.addCustomizedTrialJob('hyperParam').then(() => { + nniManager.getTrialJobStatistics().then(function (trialJobStatistics) { + if (trialJobStatistics[0].trialJobStatus === 'WAITING') + expect(trialJobStatistics[0].trialJobNumber).to.be.equal(2); + else + expect(trialJobStatistics[1].trialJobNumber).to.be.equal(2); + }) + }).catch((error) => { + assert.fail(error); + }) + }) + + it('test resumeExperiment', async () => { + //TODO: add resume experiment unit test + }) + }) diff --git a/src/nni_manager/package.json b/src/nni_manager/package.json index 757677211a..4d5c864d89 100644 --- a/src/nni_manager/package.json +++ b/src/nni_manager/package.json @@ -5,7 +5,7 @@ "scripts": { "postbuild": "cp -rf config ./dist/", "build": "tsc", - "test": "nyc mocha -r ts-node/register -t 15000 --recursive **/*.test.ts --exclude node_modules/**/**/*.test.ts --exclude core/test/nnimanager.test.ts --colors", + "test": "nyc mocha -r ts-node/register -t 15000 --recursive **/*.test.ts --exclude node_modules/**/**/*.test.ts --colors", "start": "node dist/main.js", "tslint": "tslint -p ." }, From 9a3a75c88c8dc6e12208bf03f190d404ace6fa07 Mon Sep 17 00:00:00 2001 From: fishyds Date: Mon, 25 Feb 2019 11:40:57 +0800 Subject: [PATCH 3/9] Fix a race condition bug that does not store Trial Job cancel status correctly (#707) * Fix a race condition bug that does not store Trial Job cancel status correctly --- src/nni_manager/common/trainingService.ts | 1 + .../kubeflow/kubeflowJobInfoCollector.ts | 1 + src/nni_manager/training_service/pai/paiData.ts | 1 + .../training_service/pai/paiJobInfoCollector.ts | 8 ++++++-- .../training_service/pai/paiTrainingService.ts | 7 ++++--- .../remote_machine/remoteMachineData.ts | 1 + .../remoteMachineTrainingService.ts | 17 ++++++++++++++--- 7 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/nni_manager/common/trainingService.ts b/src/nni_manager/common/trainingService.ts index 17e232be56..fd1be13e75 100644 --- a/src/nni_manager/common/trainingService.ts +++ b/src/nni_manager/common/trainingService.ts @@ -71,6 +71,7 @@ interface TrialJobDetail { readonly workingDirectory: string; readonly form: JobApplicationForm; readonly sequenceId: number; + isEarlyStopped?: boolean; } interface HostJobDetail { diff --git a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts index 84fd313001..be4468c455 100644 --- a/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts +++ b/src/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts @@ -46,6 +46,7 @@ export class KubeflowJobInfoCollector extends KubernetesJobInfoCollector{ try { kubernetesJobInfo = await kubernetesCRDClient.getKubernetesJob(kubernetesTrialJob.kubernetesJobName); } catch(error) { + // Notice: it maynot be a 'real' error since cancel trial job can also cause getKubernetesJob failed. this.log.error(`Get job ${kubernetesTrialJob.kubernetesJobName} info failed, error is ${error}`); //This is not treat as a error status return Promise.resolve(); diff --git a/src/nni_manager/training_service/pai/paiData.ts b/src/nni_manager/training_service/pai/paiData.ts index 036c206c68..0f1923009e 100644 --- a/src/nni_manager/training_service/pai/paiData.ts +++ b/src/nni_manager/training_service/pai/paiData.ts @@ -34,6 +34,7 @@ export class PAITrialJobDetail implements TrialJobDetail { public form: JobApplicationForm; public sequenceId: number; public hdfsLogPath: string; + public isEarlyStopped?: boolean; constructor(id: string, status: TrialJobStatus, paiJobName : string, submitTime: number, workingDirectory: string, form: JobApplicationForm, sequenceId: number, hdfsLogPath: string) { diff --git a/src/nni_manager/training_service/pai/paiJobInfoCollector.ts b/src/nni_manager/training_service/pai/paiJobInfoCollector.ts index a4540809b8..5fbffcb9e9 100644 --- a/src/nni_manager/training_service/pai/paiJobInfoCollector.ts +++ b/src/nni_manager/training_service/pai/paiJobInfoCollector.ts @@ -103,8 +103,12 @@ export class PAIJobInfoCollector { paiTrialJob.status = 'SUCCEEDED'; break; case 'STOPPED': - if (paiTrialJob.status !== 'EARLY_STOPPED') { - paiTrialJob.status = 'USER_CANCELED'; + if (paiTrialJob.isEarlyStopped !== undefined) { + paiTrialJob.status = paiTrialJob.isEarlyStopped === true ? + 'EARLY_STOPPED' : 'USER_CANCELED'; + } else { + // if paiTrialJob's isEarlyStopped is undefined, that mean we didn't stop it via cancellation, mark it as SYS_CANCELLED by PAI + paiTrialJob.status = 'SYS_CANCELED'; } break; case 'FAILED': diff --git a/src/nni_manager/training_service/pai/paiTrainingService.ts b/src/nni_manager/training_service/pai/paiTrainingService.ts index edd6b68869..88264503e6 100644 --- a/src/nni_manager/training_service/pai/paiTrainingService.ts +++ b/src/nni_manager/training_service/pai/paiTrainingService.ts @@ -324,14 +324,15 @@ class PAITrainingService implements TrainingService { "Authorization": 'Bearer ' + this.paiToken } }; + + // Set trialjobDetail's early stopped field, to mark the job's cancellation source + trialJobDetail.isEarlyStopped = isEarlyStopped; + request(stopJobRequest, (error: Error, response: request.Response, body: any) => { if (error || response.statusCode >= 400) { this.log.error(`PAI Training service: stop trial ${trialJobId} to PAI Cluster failed!`); deferred.reject(error ? error.message : 'Stop trial failed, http code: ' + response.statusCode); } else { - if (isEarlyStopped) { - trialJobDetail.status = 'EARLY_STOPPED'; - } deferred.resolve(); } }); diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts index fdedd78888..1619158794 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts @@ -80,6 +80,7 @@ export class RemoteMachineTrialJobDetail implements TrialJobDetail { public form: JobApplicationForm; public sequenceId: number; public rmMeta?: RemoteMachineMeta; + public isEarlyStopped?: boolean; constructor(id: string, status: TrialJobStatus, submitTime: number, workingDirectory: string, form: JobApplicationForm, sequenceId: number) { diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts index 50bb6ab4ef..794f667e77 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts @@ -48,7 +48,7 @@ import { GPU_COLLECTOR_FORMAT } from './remoteMachineData'; import { SSHClientUtility } from './sshClientUtility'; -import { validateCodeDir} from '../common/util'; +import { validateCodeDir } from '../common/util'; import { RemoteMachineJobRestServer } from './remoteMachineJobRestServer'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData'; import { mkDirP } from '../../common/utils'; @@ -279,8 +279,9 @@ class RemoteMachineTrainingService implements TrainingService { const jobpidPath: string = this.getJobPidPath(trialJob.id); try { + // Mark the toEarlyStop tag here + trialJob.isEarlyStopped = isEarlyStopped; await SSHClientUtility.remoteExeCommand(`pkill -P \`cat ${jobpidPath}\``, sshClient); - trialJob.status = getJobCancelStatus(isEarlyStopped); } catch (error) { // Not handle the error since pkill failed will not impact trial job's current status this.log.error(`remoteTrainingService.cancelTrialJob: ${error.message}`); @@ -482,6 +483,11 @@ class RemoteMachineTrainingService implements TrainingService { if (trialJobDetail === undefined) { throw new NNIError(NNIErrorNames.INVALID_JOB_DETAIL, `Invalid job detail information for trial job ${trialJobId}`); } + // If job is not WATIING, Don't prepare and resolve true immediately + if (trialJobDetail.status !== 'WAITING') { + deferred.resolve(true); + return deferred.promise; + } // get an ssh client from scheduler const rmScheduleResult: RemoteMachineScheduleResult = this.gpuScheduler.scheduleMachine(this.trialConfig.gpuNum, trialJobId); if (rmScheduleResult.resultType === ScheduleResultType.REQUIRE_EXCEED_TOTAL) { @@ -640,7 +646,12 @@ class RemoteMachineTrainingService implements TrainingService { if (parseInt(code, 10) === 0) { trialJob.status = 'SUCCEEDED'; } else { - trialJob.status = 'FAILED'; + // isEarlyStopped is never set, mean it's not cancelled by NNI, so if the process's exit code >0, mark it as FAILED + if (trialJob.isEarlyStopped === undefined) { + trialJob.status = 'FAILED'; + } else { + trialJob.status = getJobCancelStatus(trialJob.isEarlyStopped); + } } trialJob.endTime = parseInt(timestamp, 10); } From b8e31971e12a0675caa5d3cbfece43cf36b8dd07 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Mon, 25 Feb 2019 13:56:37 +0800 Subject: [PATCH 4/9] Support remote trialkeeper_log (#763) * add trialkeeper_stdout and trialkeeper_stderr * fix nnictl set remote nniManagerIP --- .../training_service/remote_machine/remoteMachineData.ts | 2 +- tools/nni_cmd/launcher.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts index 1619158794..68d9a2fc36 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineData.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineData.ts @@ -115,7 +115,7 @@ export NNI_PLATFORM=remote NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={ cd $NNI_SYS_DIR sh install_nni.sh echo $$ >{6} -python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' +python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr echo $? \`date +%s%3N\` >{10}`; export const HOST_JOB_SHELL_FORMAT: string = diff --git a/tools/nni_cmd/launcher.py b/tools/nni_cmd/launcher.py index fbb7d68a91..2c2b505940 100644 --- a/tools/nni_cmd/launcher.py +++ b/tools/nni_cmd/launcher.py @@ -168,7 +168,9 @@ def set_remote_config(experiment_config, port, config_file_name): with open(stderr_full_path, 'a+') as fout: fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':'))) return False, err_message - + result, message = setNNIManagerIp(experiment_config, port, config_file_name) + if not result: + return result, message #set trial_config return set_trial_config(experiment_config, port, config_file_name), err_message From 6d495c42b6285c8ca2c6bfc135db9f6413004de5 Mon Sep 17 00:00:00 2001 From: Yan Ni Date: Mon, 25 Feb 2019 14:18:26 +0800 Subject: [PATCH 5/9] move readmes for tuners under docs directory (#773) * move tuners readme under docs * fix deadlink in builtin * fix reference warning * remove smac dependency --- docs/en_US/Builtin_Assessors.md | 8 +++-- docs/en_US/Builtin_Tuner.md | 32 +++++++++++++------ docs/en_US/Reference.rst | 1 + .../README.md => docs/en_US/batchTuner.md | 2 +- docs/en_US/conf.py | 3 +- .../en_US/curvefittingAssessor.md | 0 .../README.md => docs/en_US/evolutionTuner.md | 0 .../en_US/gridsearchTuner.md | 2 +- .../en_US/hyperbandAdvisor.md | 0 .../README.md => docs/en_US/hyperoptTuner.md | 0 .../en_US/medianstopAssessor.md | 0 .../README.md => docs/en_US/metisTuner.md | 0 .../en_US/networkmorphismTuner.md | 2 +- .../README.md => docs/en_US/smacTuner.md | 2 +- docs/requirements.txt | 4 +-- 15 files changed, 35 insertions(+), 21 deletions(-) rename src/sdk/pynni/nni/batch_tuner/README.md => docs/en_US/batchTuner.md (78%) rename src/sdk/pynni/nni/curvefitting_assessor/README.md => docs/en_US/curvefittingAssessor.md (100%) rename src/sdk/pynni/nni/evolution_tuner/README.md => docs/en_US/evolutionTuner.md (100%) rename src/sdk/pynni/nni/gridsearch_tuner/README.md => docs/en_US/gridsearchTuner.md (67%) rename src/sdk/pynni/nni/hyperband_advisor/README.md => docs/en_US/hyperbandAdvisor.md (100%) rename src/sdk/pynni/nni/hyperopt_tuner/README.md => docs/en_US/hyperoptTuner.md (100%) rename src/sdk/pynni/nni/medianstop_assessor/README.md => docs/en_US/medianstopAssessor.md (100%) rename src/sdk/pynni/nni/metis_tuner/README.md => docs/en_US/metisTuner.md (100%) rename src/sdk/pynni/nni/networkmorphism_tuner/README.md => docs/en_US/networkmorphismTuner.md (98%) rename src/sdk/pynni/nni/smac_tuner/README.md => docs/en_US/smacTuner.md (78%) diff --git a/docs/en_US/Builtin_Assessors.md b/docs/en_US/Builtin_Assessors.md index 2723c35eb9..479be0da9a 100644 --- a/docs/en_US/Builtin_Assessors.md +++ b/docs/en_US/Builtin_Assessors.md @@ -4,10 +4,14 @@ NNI provides state-of-the-art tuning algorithm in our builtin-assessors and make Note: Click the **Assessor's name** to get a detailed description of the algorithm, click the corresponding **Usage** to get the Assessor's installation requirements, suggested scenario and using example. +Currently we support the following Assessors: +* [Medianstop](medianstopAssessor.md) +* [Curvefitting](curvefittingAssessor.md) + |Assessor|Brief Introduction of Algorithm| |---|---| -|[Medianstop](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/medianstop_assessor/README.md) [(Usage)](#MedianStop)|Medianstop is a simple early stopping rule. It stops a pending trial X at step S if the trial’s best objective value by step S is strictly worse than the median value of the running averages of all completed trials’ objectives reported up to step S. [Reference Paper](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/46180.pdf)| -|[Curvefitting](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/curvefitting_assessor/README.md) [(Usage)](#Curvefitting)|Curve Fitting Assessor is a LPA(learning, predicting, assessing) algorithm. It stops a pending trial X at step S if the prediction of final epoch's performance worse than the best final performance in the trial history. In this algorithm, we use 12 curves to fit the accuracy curve. [Reference Paper](http://aad.informatik.uni-freiburg.de/papers/15-IJCAI-Extrapolation_of_Learning_Curves.pdf)| +|__Medianstop__[(Usage)](#MedianStop)|Medianstop is a simple early stopping rule. It stops a pending trial X at step S if the trial’s best objective value by step S is strictly worse than the median value of the running averages of all completed trials’ objectives reported up to step S. [Reference Paper](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/46180.pdf)| +|__Curvefitting__[(Usage)](#Curvefitting)|Curve Fitting Assessor is a LPA(learning, predicting, assessing) algorithm. It stops a pending trial X at step S if the prediction of final epoch's performance worse than the best final performance in the trial history. In this algorithm, we use 12 curves to fit the accuracy curve. [Reference Paper](http://aad.informatik.uni-freiburg.de/papers/15-IJCAI-Extrapolation_of_Learning_Curves.pdf)| ## Usage of Builtin Assessors diff --git a/docs/en_US/Builtin_Tuner.md b/docs/en_US/Builtin_Tuner.md index 18dc25e7ba..be5d7445af 100644 --- a/docs/en_US/Builtin_Tuner.md +++ b/docs/en_US/Builtin_Tuner.md @@ -4,18 +4,30 @@ NNI provides state-of-the-art tuning algorithm as our builtin-tuners and makes t Note: Click the **Tuner's name** to get a detailed description of the algorithm, click the corresponding **Usage** to get the Tuner's installation requirements, suggested scenario and using example. +Currently we support the following algorithms: +* [TPE](hyperoptTuner.md) +* [Random Search](hyperoptTuner.md) +* [Anneal](hyperoptTuner.md) +* [Naive Evolution](evolutionTuner.md) +* [SMAC](smacTuner.md) +* [Batch tuner](batchTuner.md) +* [Grid Search](gridsearchTuner.md) +* [Hyperband](hyperbandAdvisor.md) +* [Network Morphism](networkmorphismTuner.md) +* [Metis Tuner](metisTuner.md) + |Tuner|Brief Introduction of Algorithm| |---|---| -|[TPE](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/hyperopt_tuner/README.md) [(Usage)](#TPE)|The Tree-structured Parzen Estimator (TPE) is a sequential model-based optimization (SMBO) approach. SMBO methods sequentially construct models to approximate the performance of hyperparameters based on historical measurements, and then subsequently choose new hyperparameters to test based on this model. [Reference Paper](https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf)| -|[Random Search](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/hyperopt_tuner/README.md) [(Usage)](#Random)|In Random Search for Hyper-Parameter Optimization show that Random Search might be surprisingly simple and effective. We suggest that we could use Random Search as the baseline when we have no knowledge about the prior distribution of hyper-parameters. [Reference Paper](http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf)| -|[Anneal](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/hyperopt_tuner/README.md) [(Usage)](#Anneal)|This simple annealing algorithm begins by sampling from the prior, but tends over time to sample from points closer and closer to the best ones observed. This algorithm is a simple variation on the random search that leverages smoothness in the response surface. The annealing rate is not adaptive.| -|[Naive Evolution](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/evolution_tuner/README.md) [(Usage)](#Evolution)|Naive Evolution comes from Large-Scale Evolution of Image Classifiers. It randomly initializes a population-based on search space. For each generation, it chooses better ones and does some mutation (e.g., change a hyperparameter, add/remove one layer) on them to get the next generation. Naive Evolution requires many trials to works, but it's very simple and easy to expand new features. [Reference paper](https://arxiv.org/pdf/1703.01041.pdf)| -|[SMAC](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/smac_tuner/README.md) [(Usage)](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by nni is a wrapper on the SMAC3 Github repo. Notice, SMAC need to be installed by `nnictl package` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [Github Repo](https://github.com/automl/SMAC3)| -|[Batch tuner](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/batch_tuner/README.md) [(Usage)](#Batch)|Batch tuner allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. Batch tuner only supports the type choice in search space spec.| -|[Grid Search](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/gridsearch_tuner/README.md) [(Usage)](#GridSearch)|Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are choice, quniform, qloguniform. The number q in quniform and qloguniform has special meaning (different from the spec in search space spec). It means the number of values that will be sampled evenly from the range low and high.| -|[Hyperband](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/hyperband_advisor/README.md) [(Usage)](#Hyperband)|Hyperband tries to use the limited resource to explore as many configurations as possible, and finds out the promising ones to get the final result. The basic idea is generating many configurations and to run them for the small number of STEPs to find out promising one, then further training those promising ones to select several more promising one.[Reference Paper](https://arxiv.org/pdf/1603.06560.pdf)| -|[Network Morphism](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/networkmorphism_tuner/README.md) [(Usage)](#NetworkMorphism)|Network Morphism provides functions to automatically search for architecture of deep learning models. Every child network inherits the knowledge from its parent network and morphs into diverse types of networks, including changes of depth, width, and skip-connection. Next, it estimates the value of a child network using the historic architecture and metric pairs. Then it selects the most promising one to train. [Reference Paper](https://arxiv.org/abs/1806.10282)| -|[Metis Tuner](https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/metis_tuner/README.md) [(Usage)](#MetisTuner)|Metis offers the following benefits when it comes to tuning parameters: While most tools only predict the optimal configuration, Metis gives you two outputs: (a) current prediction of optimal configuration, and (b) suggestion for the next trial. No more guesswork. While most tools assume training datasets do not have noisy data, Metis actually tells you if you need to re-sample a particular hyper-parameter. [Reference Paper](https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/)| +|__TPE__ [(Usage)](#TPE)|The Tree-structured Parzen Estimator (TPE) is a sequential model-based optimization (SMBO) approach. SMBO methods sequentially construct models to approximate the performance of hyperparameters based on historical measurements, and then subsequently choose new hyperparameters to test based on this model. [Reference Paper](https://papers.nips.cc/paper/4443-algorithms-for-hyper-parameter-optimization.pdf)| +|__Random Search__ [(Usage)](#Random)|In Random Search for Hyper-Parameter Optimization show that Random Search might be surprisingly simple and effective. We suggest that we could use Random Search as the baseline when we have no knowledge about the prior distribution of hyper-parameters. [Reference Paper](http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf)| +|__Anneal__ [(Usage)](#Anneal)|This simple annealing algorithm begins by sampling from the prior, but tends over time to sample from points closer and closer to the best ones observed. This algorithm is a simple variation on the random search that leverages smoothness in the response surface. The annealing rate is not adaptive.| +|__Naive Evolution__ [(Usage)](#Evolution)|Naive Evolution comes from Large-Scale Evolution of Image Classifiers. It randomly initializes a population-based on search space. For each generation, it chooses better ones and does some mutation (e.g., change a hyperparameter, add/remove one layer) on them to get the next generation. Naive Evolution requires many trials to works, but it's very simple and easy to expand new features. [Reference paper](https://arxiv.org/pdf/1703.01041.pdf)| +|__SMAC__ [(Usage)](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by nni is a wrapper on the SMAC3 Github repo. Notice, SMAC need to be installed by `nnictl package` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [Github Repo](https://github.com/automl/SMAC3)| +|__Batch tuner__ [(Usage)](#Batch)|Batch tuner allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. Batch tuner only supports the type choice in search space spec.| +|__Grid Search__ [(Usage)](#GridSearch)|Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are choice, quniform, qloguniform. The number q in quniform and qloguniform has special meaning (different from the spec in search space spec). It means the number of values that will be sampled evenly from the range low and high.| +|__Hyperband__ [(Usage)](#Hyperband)|Hyperband tries to use the limited resource to explore as many configurations as possible, and finds out the promising ones to get the final result. The basic idea is generating many configurations and to run them for the small number of STEPs to find out promising one, then further training those promising ones to select several more promising one.[Reference Paper](https://arxiv.org/pdf/1603.06560.pdf)| +|__Network Morphism__ [(Usage)](#NetworkMorphism)|Network Morphism provides functions to automatically search for architecture of deep learning models. Every child network inherits the knowledge from its parent network and morphs into diverse types of networks, including changes of depth, width, and skip-connection. Next, it estimates the value of a child network using the historic architecture and metric pairs. Then it selects the most promising one to train. [Reference Paper](https://arxiv.org/abs/1806.10282)| +|__Metis Tuner__ [(Usage)](#MetisTuner)|Metis offers the following benefits when it comes to tuning parameters: While most tools only predict the optimal configuration, Metis gives you two outputs: (a) current prediction of optimal configuration, and (b) suggestion for the next trial. No more guesswork. While most tools assume training datasets do not have noisy data, Metis actually tells you if you need to re-sample a particular hyper-parameter. [Reference Paper](https://www.microsoft.com/en-us/research/publication/metis-robustly-tuning-tail-latencies-cloud-systems/)|
diff --git a/docs/en_US/Reference.rst b/docs/en_US/Reference.rst index df57d3af6e..3ce330c3c4 100644 --- a/docs/en_US/Reference.rst +++ b/docs/en_US/Reference.rst @@ -5,6 +5,7 @@ References :maxdepth: 3 Command Line + Python API Annotation Configuration Search Space \ No newline at end of file diff --git a/src/sdk/pynni/nni/batch_tuner/README.md b/docs/en_US/batchTuner.md similarity index 78% rename from src/sdk/pynni/nni/batch_tuner/README.md rename to docs/en_US/batchTuner.md index ed8ea1dbe8..c949eea5df 100644 --- a/src/sdk/pynni/nni/batch_tuner/README.md +++ b/docs/en_US/batchTuner.md @@ -3,6 +3,6 @@ Batch Tuner on NNI ## Batch Tuner -Batch tuner allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. Batch tuner only supports the type choice in [search space spec](../../../../../docs/en_US/SearchSpaceSpec.md). +Batch tuner allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. Batch tuner only supports the type choice in [search space spec](SearchSpaceSpec.md). Suggested sceanrio: If the configurations you want to try have been decided, you can list them in searchspace file (using choice) and run them using batch tuner. \ No newline at end of file diff --git a/docs/en_US/conf.py b/docs/en_US/conf.py index 77f3a1a395..65cfa4eb14 100644 --- a/docs/en_US/conf.py +++ b/docs/en_US/conf.py @@ -189,7 +189,6 @@ github_doc_root = 'https://github.com/Microsoft/nni/tree/master/doc/' def setup(app): app.add_config_value('recommonmark_config', { - 'url_resolver': lambda url: github_doc_root + url if url.startswith('..') else url, - 'enable_auto_toc_tree': False, + 'enable_auto_toc_tree': True, }, True) app.add_transform(AutoStructify) diff --git a/src/sdk/pynni/nni/curvefitting_assessor/README.md b/docs/en_US/curvefittingAssessor.md similarity index 100% rename from src/sdk/pynni/nni/curvefitting_assessor/README.md rename to docs/en_US/curvefittingAssessor.md diff --git a/src/sdk/pynni/nni/evolution_tuner/README.md b/docs/en_US/evolutionTuner.md similarity index 100% rename from src/sdk/pynni/nni/evolution_tuner/README.md rename to docs/en_US/evolutionTuner.md diff --git a/src/sdk/pynni/nni/gridsearch_tuner/README.md b/docs/en_US/gridsearchTuner.md similarity index 67% rename from src/sdk/pynni/nni/gridsearch_tuner/README.md rename to docs/en_US/gridsearchTuner.md index 5f1914b608..9f858b4d88 100644 --- a/src/sdk/pynni/nni/gridsearch_tuner/README.md +++ b/docs/en_US/gridsearchTuner.md @@ -3,4 +3,4 @@ Grid Search on NNI ## Grid Search -Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are `choice`, `quniform`, `qloguniform`. **The number `q` in `quniform` and `qloguniform` has special meaning (different from the spec in [search space spec](../../../../../docs/en_US/SearchSpaceSpec.md)). It means the number of values that will be sampled evenly from the range `low` and `high`.** \ No newline at end of file +Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are `choice`, `quniform`, `qloguniform`. **The number `q` in `quniform` and `qloguniform` has special meaning (different from the spec in [search space spec](SearchSpaceSpec.md)). It means the number of values that will be sampled evenly from the range `low` and `high`.** \ No newline at end of file diff --git a/src/sdk/pynni/nni/hyperband_advisor/README.md b/docs/en_US/hyperbandAdvisor.md similarity index 100% rename from src/sdk/pynni/nni/hyperband_advisor/README.md rename to docs/en_US/hyperbandAdvisor.md diff --git a/src/sdk/pynni/nni/hyperopt_tuner/README.md b/docs/en_US/hyperoptTuner.md similarity index 100% rename from src/sdk/pynni/nni/hyperopt_tuner/README.md rename to docs/en_US/hyperoptTuner.md diff --git a/src/sdk/pynni/nni/medianstop_assessor/README.md b/docs/en_US/medianstopAssessor.md similarity index 100% rename from src/sdk/pynni/nni/medianstop_assessor/README.md rename to docs/en_US/medianstopAssessor.md diff --git a/src/sdk/pynni/nni/metis_tuner/README.md b/docs/en_US/metisTuner.md similarity index 100% rename from src/sdk/pynni/nni/metis_tuner/README.md rename to docs/en_US/metisTuner.md diff --git a/src/sdk/pynni/nni/networkmorphism_tuner/README.md b/docs/en_US/networkmorphismTuner.md similarity index 98% rename from src/sdk/pynni/nni/networkmorphism_tuner/README.md rename to docs/en_US/networkmorphismTuner.md index 98accf0adb..f311a2e8f7 100644 --- a/src/sdk/pynni/nni/networkmorphism_tuner/README.md +++ b/docs/en_US/networkmorphismTuner.md @@ -4,7 +4,7 @@ [Autokeras](https://arxiv.org/abs/1806.10282) is a popular automl tools using Network Morphism. The basic idea of Autokeras is to use Bayesian Regression to estimate the metric of the Neural Network Architecture. Each time, it generates several child networks from father networks. Then it uses a naïve Bayesian regression estimate its metric value from history trained results of network and metric value pair. Next, it chooses the the child which has best estimated performance and adds it to the training queue. Inspired by its work and referring to its [code](https://github.com/jhfjhfj1/autokeras), we implement our Network Morphism method in our NNI platform. -If you want to know about network morphism trial usage, please check [Readme.md](../../../../../examples/trials/network_morphism/README.md) of the trial to get more detail. +If you want to know about network morphism trial usage, please check [Readme.md](https://github.com/Microsoft/nni/blob/master/examples/trials/network_morphism/README.md) of the trial to get more detail. ## 2. Usage diff --git a/src/sdk/pynni/nni/smac_tuner/README.md b/docs/en_US/smacTuner.md similarity index 78% rename from src/sdk/pynni/nni/smac_tuner/README.md rename to docs/en_US/smacTuner.md index e82d19b5a8..7a43deb306 100644 --- a/src/sdk/pynni/nni/smac_tuner/README.md +++ b/docs/en_US/smacTuner.md @@ -5,4 +5,4 @@ SMAC Tuner on NNI [SMAC](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by nni is a wrapper on [the SMAC3 github repo](https://github.com/automl/SMAC3). -Note that SMAC on nni only supports a subset of the types in [search space spec](../../../../../docs/en_US/SearchSpaceSpec.md), including `choice`, `randint`, `uniform`, `loguniform`, `quniform(q=1)`. \ No newline at end of file +Note that SMAC on nni only supports a subset of the types in [search space spec](SearchSpaceSpec.md), including `choice`, `randint`, `uniform`, `loguniform`, `quniform(q=1)`. \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index e57099ebe3..2079b87621 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -8,6 +8,4 @@ hyperopt json_tricks numpy scipy -coverage -git+https://github.com/QuanluZhang/ConfigSpace.git -git+https://github.com/QuanluZhang/SMAC3.git +coverage \ No newline at end of file From 8c4c0ef2410317723622fe00a35e7a9c0f6544f9 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Mon, 25 Feb 2019 14:48:17 +0800 Subject: [PATCH 6/9] Support webhdfs path in python hdfs client (#722) trial_keeper use 50070 port to connect to webhdfs server, and PAI use a mapping method to map 50070 port to 5070 port to visit restful server, this method has some risk for PAI may not support this kind of mapping in later release.Now use Pylon path(/webhdfs/api/v1) instead of 50070 port in webhdfs client of trial_keeper, the path is transmitted in trainingService. In this pr, we have these changes: 1. Change to use webhdfs path instead of 50070 port in hdfs client. 2. Change to use new hdfs package "PythonWebHDFS", which is build to support pylon by myself. You could test the new function from "sparksnail/nni:dev-pai" image to test pai trainingService. 3. Update some variables' name according to comments. --- deployment/pypi/setup.py | 2 +- setup.py | 2 +- .../training_service/pai/paiData.ts | 2 +- tools/nni_trial_tool/trial_keeper.py | 34 ++++++++++++++----- tools/setup.py | 2 +- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/deployment/pypi/setup.py b/deployment/pypi/setup.py index bd97927c3f..c0447b4f70 100644 --- a/deployment/pypi/setup.py +++ b/deployment/pypi/setup.py @@ -63,7 +63,7 @@ 'psutil', 'requests', 'astor', - 'pyhdfs', + 'PythonWebHDFS', 'hyperopt', 'json_tricks', 'numpy', diff --git a/setup.py b/setup.py index d9329bd259..a543a09250 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ def run(self): 'requests', 'scipy', 'schema', - 'pyhdfs' + 'PythonWebHDFS' ], cmdclass={ diff --git a/src/nni_manager/training_service/pai/paiData.ts b/src/nni_manager/training_service/pai/paiData.ts index 0f1923009e..ed9fb8d235 100644 --- a/src/nni_manager/training_service/pai/paiData.ts +++ b/src/nni_manager/training_service/pai/paiData.ts @@ -64,7 +64,7 @@ export const PAI_TRIAL_COMMAND_FORMAT: string = `export NNI_PLATFORM=pai NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} && cd $NNI_SYS_DIR && sh install_nni.sh && python3 -m nni_trial_tool.trial_keeper --trial_command '{5}' --nnimanager_ip '{6}' --nnimanager_port '{7}' ---pai_hdfs_output_dir '{8}' --pai_hdfs_host '{9}' --pai_user_name {10} --nni_hdfs_exp_dir '{11}'`; +--pai_hdfs_output_dir '{8}' --pai_hdfs_host '{9}' --pai_user_name {10} --nni_hdfs_exp_dir '{11}' --webhdfs_path '/webhdfs/api/v1'`; export const PAI_OUTPUT_DIR_FORMAT: string = `hdfs://{0}:9000/`; diff --git a/tools/nni_trial_tool/trial_keeper.py b/tools/nni_trial_tool/trial_keeper.py index 1675f8bf49..2d9837a30e 100644 --- a/tools/nni_trial_tool/trial_keeper.py +++ b/tools/nni_trial_tool/trial_keeper.py @@ -48,10 +48,25 @@ def main_loop(args): # redirect trial keeper's stdout and stderr to syslog trial_syslogger_stdout = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial', StdOutputType.Stdout) sys.stdout = sys.stderr = trial_keeper_syslogger + # backward compatibility + hdfs_host = None + hdfs_output_dir = None + if args.hdfs_host: + hdfs_host = args.hdfs_host + elif args.pai_hdfs_host: + hdfs_host = args.pai_hdfs_host + if args.hdfs_output_dir: + hdfs_output_dir = args.hdfs_output_dir + elif args.pai_hdfs_output_dir: + hdfs_output_dir = args.pai_hdfs_output_dir - if args.pai_hdfs_host is not None and args.nni_hdfs_exp_dir is not None: + if hdfs_host is not None and args.nni_hdfs_exp_dir is not None: try: - hdfs_client = HdfsClient(hosts='{0}:{1}'.format(args.pai_hdfs_host, '50070'), user_name=args.pai_user_name, timeout=5) + if args.webhdfs_path: + hdfs_client = HdfsClient(hosts='{0}:80'.format(hdfs_host), user_name=args.pai_user_name, webhdfs_path=args.webhdfs_path, timeout=5) + else: + # backward compatibility + hdfs_client = HdfsClient(hosts='{0}:{1}'.format(hdfs_host, '50070'), user_name=args.pai_user_name, timeout=5) except Exception as e: nni_log(LogType.Error, 'Create HDFS client error: ' + str(e)) raise e @@ -67,14 +82,14 @@ def main_loop(args): # child worker process exits and all stdout data is read if retCode is not None and log_pipe_stdout.set_process_exit() and log_pipe_stdout.is_read_completed == True: nni_log(LogType.Info, 'subprocess terminated. Exit code is {}. Quit'.format(retCode)) - if args.pai_hdfs_output_dir is not None: + if hdfs_output_dir is not None: # Copy local directory to hdfs for OpenPAI nni_local_output_dir = os.environ['NNI_OUTPUT_DIR'] try: - if copyDirectoryToHdfs(nni_local_output_dir, args.pai_hdfs_output_dir, hdfs_client): - nni_log(LogType.Info, 'copy directory from {0} to {1} success!'.format(nni_local_output_dir, args.pai_hdfs_output_dir)) + if copyDirectoryToHdfs(nni_local_output_dir, hdfs_output_dir, hdfs_client): + nni_log(LogType.Info, 'copy directory from {0} to {1} success!'.format(nni_local_output_dir, hdfs_output_dir)) else: - nni_log(LogType.Info, 'copy directory from {0} to {1} failed!'.format(nni_local_output_dir, args.pai_hdfs_output_dir)) + nni_log(LogType.Info, 'copy directory from {0} to {1} failed!'.format(nni_local_output_dir, hdfs_output_dir)) except Exception as e: nni_log(LogType.Error, 'HDFS copy directory got exception: ' + str(e)) raise e @@ -95,10 +110,13 @@ def trial_keeper_help_info(*args): PARSER.add_argument('--trial_command', type=str, help='Command to launch trial process') PARSER.add_argument('--nnimanager_ip', type=str, default='localhost', help='NNI manager rest server IP') PARSER.add_argument('--nnimanager_port', type=str, default='8081', help='NNI manager rest server port') - PARSER.add_argument('--pai_hdfs_output_dir', type=str, help='the output dir of hdfs') - PARSER.add_argument('--pai_hdfs_host', type=str, help='the host of hdfs') + PARSER.add_argument('--pai_hdfs_output_dir', type=str, help='the output dir of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_output_dir', type=str, help='the output dir of hdfs') + PARSER.add_argument('--pai_hdfs_host', type=str, help='the host of pai_hdfs') # backward compatibility + PARSER.add_argument('--hdfs_host', type=str, help='the host of hdfs') PARSER.add_argument('--pai_user_name', type=str, help='the username of hdfs') PARSER.add_argument('--nni_hdfs_exp_dir', type=str, help='nni experiment directory in hdfs') + PARSER.add_argument('--webhdfs_path', type=str, help='the webhdfs path used in webhdfs URL') args, unknown = PARSER.parse_known_args() if args.trial_command is None: exit(1) diff --git a/tools/setup.py b/tools/setup.py index 9e475a59aa..f015876e67 100644 --- a/tools/setup.py +++ b/tools/setup.py @@ -12,7 +12,7 @@ 'psutil', 'astor', 'schema', - 'pyhdfs' + 'PythonWebHDFS' ], author = 'Microsoft NNI Team', From 9c5c52b9b2962733268f13bc957ba1eef734ece8 Mon Sep 17 00:00:00 2001 From: Lijiao <35484733+lvybriage@users.noreply.github.com> Date: Mon, 25 Feb 2019 15:02:52 +0800 Subject: [PATCH 7/9] [WebUI] change version in feeback link (#777) * change version --- src/webui/src/components/SlideBar.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webui/src/components/SlideBar.tsx b/src/webui/src/components/SlideBar.tsx index 7b758ebd27..a5e960b493 100644 --- a/src/webui/src/components/SlideBar.tsx +++ b/src/webui/src/components/SlideBar.tsx @@ -182,6 +182,7 @@ class SlideBar extends React.Component<{}, SliderState> { render() { const { version, menuVisible } = this.state; + const feed = `https://github.com/Microsoft/nni/issues/new?labels=${version}`; const menu = (

Experiment Parameters @@ -221,7 +222,7 @@ class SlideBar extends React.Component<{}, SliderState> { Download - + NNI github issue Date: Mon, 25 Feb 2019 15:31:41 +0800 Subject: [PATCH 8/9] Set pai/kubeflow integration test timeout to infinite (#778) Set pai/kubeflow integration test job timeout to infinite. --- test/config_test.py | 6 +++--- test/pipelines-it-kubeflow.yml | 1 + test/pipelines-it-pai.yml | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/config_test.py b/test/config_test.py index 2fdc2dff27..dece4cf490 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -38,7 +38,6 @@ def gen_new_config(config_file, training_service='local'): new_config_file = config_file + '.tmp' ts = get_yml_content('training_service.yml')[training_service] - print(config) print(ts) # hack for kubeflow trial config @@ -64,7 +63,6 @@ def run_test(config_file, training_service, local_gpu=False): return try: - print('Testing %s...' % config_file) proc = subprocess.run(['nnictl', 'create', '--config', new_config_file]) assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode @@ -109,8 +107,10 @@ def run(args): try: # sleep 5 seconds here, to make sure previous stopped exp has enough time to exit to avoid port conflict time.sleep(5) + print(GREEN + 'Testing:' + config_file + CLEAR) + begin_time = time.time() run_test(config_file, args.ts, args.local_gpu) - print(GREEN + 'Test %s: TEST PASS' % (config_file) + CLEAR) + print(GREEN + 'Test %s: TEST PASS IN %d mins' % (config_file, (time.time() - begin_time)/60) + CLEAR) except Exception as error: print(RED + 'Test %s: TEST FAIL' % (config_file) + CLEAR) print('%r' % error) diff --git a/test/pipelines-it-kubeflow.yml b/test/pipelines-it-kubeflow.yml index 0923fcafbe..198b036cca 100644 --- a/test/pipelines-it-kubeflow.yml +++ b/test/pipelines-it-kubeflow.yml @@ -1,5 +1,6 @@ jobs: - job: 'integration_test_kubeflow' + timeoutInMinutes: 0 pool: 'NNI CI KUBE CLI' variables: diff --git a/test/pipelines-it-pai.yml b/test/pipelines-it-pai.yml index 61525adf03..c0e3181df8 100644 --- a/test/pipelines-it-pai.yml +++ b/test/pipelines-it-pai.yml @@ -1,5 +1,6 @@ jobs: - job: 'integration_test_pai' + timeoutInMinutes: 0 pool: 'NNI CI PAI CLI' variables: From 51fbf6957f484b74e12f2b60bd2ca8a49345e1e4 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Mon, 25 Feb 2019 16:22:17 +0800 Subject: [PATCH 9/9] Local TrainingService UT (#772) --- .../local/localTrainingService.ts | 3 +- .../test/localTrainingService.test.ts | 102 +++++++++++++++++- 2 files changed, 99 insertions(+), 6 deletions(-) diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index a194216f35..7cd34f0c8d 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -255,7 +255,7 @@ class LocalTrainingService implements TrainingService { } if (trialJob.pid === undefined){ this.setTrialJobStatus(trialJob, 'USER_CANCELED'); - return; + return Promise.resolve(); } if (trialJob.form.jobType === 'TRIAL') { await tkill(trialJob.pid, 'SIGKILL'); @@ -265,6 +265,7 @@ class LocalTrainingService implements TrainingService { throw new Error(`Job type not supported: ${trialJob.form.jobType}`); } this.setTrialJobStatus(trialJob, getJobCancelStatus(isEarlyStopped)); + return Promise.resolve(); } public async setClusterMetadata(key: string, value: string): Promise { diff --git a/src/nni_manager/training_service/test/localTrainingService.test.ts b/src/nni_manager/training_service/test/localTrainingService.test.ts index 13210082b1..c980df997a 100644 --- a/src/nni_manager/training_service/test/localTrainingService.test.ts +++ b/src/nni_manager/training_service/test/localTrainingService.test.ts @@ -19,14 +19,106 @@ 'use strict'; -import { TrainingService } from '../../common/trainingService'; -import { LocalTrainingService } from '../local/localTrainingService'; +import * as assert from 'assert'; +import * as chai from 'chai'; +import * as chaiAsPromised from 'chai-as-promised'; +import * as fs from 'fs'; +import * as tmp from 'tmp'; import * as component from '../../common/component'; +import { TrialJobApplicationForm, TrialJobDetail, TrainingService } from '../../common/trainingService'; +import { cleanupUnitTest, delay, prepareUnitTest } from '../../common/utils'; +import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey'; +import { LocalTrainingServiceForGPU } from '../local/localTrainingServiceForGPU'; + +// TODO: copy mockedTrail.py to local folder +const localCodeDir: string = tmp.dirSync().name +const mockedTrialPath: string = './training_service/test/mockedTrial.py' +fs.copyFileSync(mockedTrialPath, localCodeDir + '/mockedTrial.py') describe('Unit Test for LocalTrainingService', () => { - let trainingService: TrainingService + let trialConfig: any = `{"command":"sleep 1h && echo hello","codeDir":"${localCodeDir}","gpuNum":1}` + + let localTrainingService: LocalTrainingServiceForGPU; + + before(() => { + chai.should(); + chai.use(chaiAsPromised); + prepareUnitTest(); + }); + + after(() => { + cleanupUnitTest(); + }); + + beforeEach(() => { + localTrainingService = component.get(LocalTrainingServiceForGPU); + localTrainingService.run(); + }); + + afterEach(() => { + localTrainingService.cleanUp(); + }); + + it('List empty trial jobs', async () => { + //trial jobs should be empty, since there are no submitted jobs + chai.expect(await localTrainingService.listTrialJobs()).to.be.empty; + }); + + it('setClusterMetadata and getClusterMetadata', async () => { + await localTrainingService.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, trialConfig); + localTrainingService.getClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG).then((data)=>{ + chai.expect(data).to.be.equals(trialConfig); + }); + }); + + it('Submit job and Cancel job', async () => { + await localTrainingService.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, trialConfig); + + // submit job + const form: TrialJobApplicationForm = { + jobType: 'TRIAL', + hyperParameters: { + value: 'mock hyperparameters', + index: 0 + } + }; + const jobDetail: TrialJobDetail = await localTrainingService.submitTrialJob(form); + chai.expect(jobDetail.status).to.be.equals('WAITING'); + await localTrainingService.cancelTrialJob(jobDetail.id); + chai.expect(jobDetail.status).to.be.equals('USER_CANCELED'); + }).timeout(20000); + + it('Read metrics, Add listener, and remove listener', async () => { + // set meta data + const trialConfig: string = `{\"command\":\"python3 mockedTrial.py\", \"codeDir\":\"${localCodeDir}\",\"gpuNum\":0}` + await localTrainingService.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, trialConfig); + + // submit job + const form: TrialJobApplicationForm = { + jobType: 'TRIAL', + hyperParameters: { + value: 'mock hyperparameters', + index: 0 + } + }; + const jobDetail: TrialJobDetail = await localTrainingService.submitTrialJob(form); + chai.expect(jobDetail.status).to.be.equals('WAITING'); + localTrainingService.listTrialJobs().then((jobList)=>{ + chai.expect(jobList.length).to.be.equals(1); + }); + // Add metrics listeners + const listener1 = function f1(metric: any) { + chai.expect(metric.id).to.be.equals(jobDetail.id); + } + localTrainingService.addTrialJobMetricListener(listener1); + // Wait to collect metric + await delay(1000); + + await localTrainingService.cancelTrialJob(jobDetail.id); + localTrainingService.removeTrialJobMetricListener(listener1); + }).timeout(20000); - beforeEach(async () => { - trainingService = component.get(LocalTrainingService); + it('Test multiphaseSupported', () => { + chai.expect(localTrainingService.isMultiPhaseJobSupported).to.be.equals(true) }) }); \ No newline at end of file