From 7410b3d75d19484ed7fe65b7c58710571e94f78a Mon Sep 17 00:00:00 2001 From: YundongYe Date: Thu, 28 Feb 2019 14:30:02 +0800 Subject: [PATCH 1/6] [Job Debugging] If user command's exitcode is none zero, container will be reserved. (#2225) --- src/rest-server/src/models/job.js | 1 + .../src/templates/dockerContainerScript.mustache | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/rest-server/src/models/job.js b/src/rest-server/src/models/job.js index e27617ba12..b275aa8539 100644 --- a/src/rest-server/src/models/job.js +++ b/src/rest-server/src/models/job.js @@ -444,6 +444,7 @@ class Job { 'azRDMA': azureEnv.azRDMA === 'false' ? false : true, 'paiMachineList': paiConfig.machineList, 'reqAzRDMA': data.jobEnvs && data.jobEnvs.paiAzRDMA === true ? true : false, + 'isDebug': data.jobEnvs && data.jobEnvs.isDebug === true ? true : false, }); return dockerContainerScript; } diff --git a/src/rest-server/src/templates/dockerContainerScript.mustache b/src/rest-server/src/templates/dockerContainerScript.mustache index fce0b9bb60..2e77cc113e 100644 --- a/src/rest-server/src/templates/dockerContainerScript.mustache +++ b/src/rest-server/src/templates/dockerContainerScript.mustache @@ -202,6 +202,15 @@ else wait $user_command_pid user_command_exitcode=$? echo "job has finished with exit code $user_command_exitcode" +{{# isDebug }} + if [[ $user_command_exitcode -ne 0 ]]; then + echo "=============================================================================" + echo "====== The job container failed, so it will be reserved for 1 week ======" + echo "====== After debugging, please stop the job manually. ======" + echo "=============================================================================" + sleep 604800 + fi +{{/ isDebug }} exit $user_command_exitcode fi From 0464c2cc4d94e13d0eacfccf176913c5a0b980d0 Mon Sep 17 00:00:00 2001 From: YundongYe Date: Fri, 1 Mar 2019 16:31:29 +0800 Subject: [PATCH 2/6] [Job Debugging] Fix Issue that user can't stop the job in status of job debugging (#2248) --- .../src/templates/dockerContainerScript.mustache | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/rest-server/src/templates/dockerContainerScript.mustache b/src/rest-server/src/templates/dockerContainerScript.mustache index 2e77cc113e..e33748d52d 100644 --- a/src/rest-server/src/templates/dockerContainerScript.mustache +++ b/src/rest-server/src/templates/dockerContainerScript.mustache @@ -208,7 +208,16 @@ else echo "====== The job container failed, so it will be reserved for 1 week ======" echo "====== After debugging, please stop the job manually. ======" echo "=============================================================================" - sleep 604800 + + sleep_time=604800 + sleep_count=0 + + while [ $(( $(date +%s) - $(stat -c %Y /alive/yarn_$PAI_CONTAINER_ID) )) -lt 30 ] && \ + [ "$sleep_count" -lt "$sleep_time" ]; do + sleep 20 + sleep_count=$((sleep_count+20)) + done + fi {{/ isDebug }} exit $user_command_exitcode From fd6039ce25cc5ffd19ef8537ccb4b127b835c0c4 Mon Sep 17 00:00:00 2001 From: YundongYe Date: Tue, 5 Mar 2019 10:06:04 +0800 Subject: [PATCH 3/6] Make debugging reservation time configurable. (#2257) --- .../cluster-configuration/services-configuration.yaml | 4 +++- src/rest-server/config/rest-server.md | 8 ++++++++ src/rest-server/config/rest-server.yaml | 1 + src/rest-server/config/rest_server.py | 7 +++++++ src/rest-server/deploy/rest-server.yaml.template | 2 ++ src/rest-server/src/config/paiConfig.js | 5 +++-- src/rest-server/src/models/job.js | 1 + .../src/templates/dockerContainerScript.mustache | 2 +- src/rest-server/test/setup.js | 1 + 9 files changed, 27 insertions(+), 4 deletions(-) diff --git a/examples/cluster-configuration/services-configuration.yaml b/examples/cluster-configuration/services-configuration.yaml index 2c6fed8bba..01e741ca4f 100644 --- a/examples/cluster-configuration/services-configuration.yaml +++ b/examples/cluster-configuration/services-configuration.yaml @@ -82,10 +82,12 @@ rest-server: default-pai-admin-username: your_default_pai_admin_username # database admin password default-pai-admin-password: your_default_pai_admin_password - # rest server would achieve marketplace template from below configed github repository + # rest server would achieve marketplace template from below configed github repository #github-owner: Microsoft #github-repository: pai #github-path: marketplace + # Job Debugging Reservation Seconds. + #debugging-reservation-seconds: 604800 # uncomment following section if you want to customize the port of web portal # webportal: diff --git a/src/rest-server/config/rest-server.md b/src/rest-server/config/rest-server.md index 4b152b46bd..d5ecdd869d 100644 --- a/src/rest-server/config/rest-server.md +++ b/src/rest-server/config/rest-server.md @@ -19,6 +19,7 @@ other config fields are optional, includes: - `github-owner: Microsoft` The marketplace repo owner in GitHub - `github-repository: pai` The marketplace repo name - `github-path: marketplace` The marketpalce path in the repo +- `debugging-reservation-seconds: 604800` The seconds to reserved a job container to debug. ## Generated Configuration @@ -34,6 +35,7 @@ rest-server: github-owner: Microsoft github-repository: pai github-path: marketplace + debugging-reservation-seconds: 604800 ``` ## Table @@ -99,4 +101,10 @@ rest-server: cluster_cfg["rest-server"]["etcd-uris"] String + + rest-server.debugging-reservation-seconds + com["rest-server"]["debugging-reservation-seconds"] + cluster_cfg["rest-server"]["debugging-reservation-seconds"] + String + diff --git a/src/rest-server/config/rest-server.yaml b/src/rest-server/config/rest-server.yaml index bfe1248ec7..46bfb6e0de 100644 --- a/src/rest-server/config/rest-server.yaml +++ b/src/rest-server/config/rest-server.yaml @@ -20,3 +20,4 @@ jwt-secret: pai-secret github-owner: Microsoft github-repository: pai github-path: marketplace +debugging-reservation-seconds: 604800 \ No newline at end of file diff --git a/src/rest-server/config/rest_server.py b/src/rest-server/config/rest_server.py index 31929d13ce..a9975960e0 100644 --- a/src/rest-server/config/rest_server.py +++ b/src/rest-server/config/rest_server.py @@ -29,6 +29,12 @@ def validation_pre(self): return False, '"default-pai-admin-username" is required in rest-server' if 'default-pai-admin-password' not in self.service_configuration: return False, '"default-pai-admin-password" is required in rest-server' + try: + reservation_time = int(self.service_configuration['debugging-reservation-seconds']) + except ValueError: + return False, '"debugging-reservation-seconds" should be a positive integer.' + if reservation_time <= 0: + return False, '"debugging-reservation-seconds" should be a positive integer.' return True, None @@ -50,6 +56,7 @@ def run(self): service_object_model['github-owner'] = self.service_configuration['github-owner'] service_object_model['github-repository'] = self.service_configuration['github-repository'] service_object_model['github-path'] = self.service_configuration['github-path'] + service_object_model['debugging-reservation-seconds'] = self.service_configuration['debugging-reservation-seconds'] return service_object_model diff --git a/src/rest-server/deploy/rest-server.yaml.template b/src/rest-server/deploy/rest-server.yaml.template index ab3039150f..339834b75b 100644 --- a/src/rest-server/deploy/rest-server.yaml.template +++ b/src/rest-server/deploy/rest-server.yaml.template @@ -57,6 +57,8 @@ spec: value: {{ cluster_cfg['layout']['kubernetes']['api-servers-url'] }} - name: AZ_RDMA value: "{{ cluster_cfg['cluster']['common']['az-rdma']}}" + - name: DEBUGGING_RESERVATION_SECONDS + value: "{{ cluster_cfg['rest-server']['debugging-reservation-seconds']}}" {% if cluster_cfg['rest-server']['github-owner'] %} - name: GITHUB_OWNER value: {{ cluster_cfg['rest-server']['github-owner'] }} diff --git a/src/rest-server/src/config/paiConfig.js b/src/rest-server/src/config/paiConfig.js index 100e4e3f1f..f8f1f3575a 100644 --- a/src/rest-server/src/config/paiConfig.js +++ b/src/rest-server/src/config/paiConfig.js @@ -26,18 +26,19 @@ try { paiMachineList = yaml.safeLoad(fs.readFileSync('/pai-cluster-config/layout.yaml', 'utf8'))['machine-list']; } catch (err) { paiMachineList = []; - logger.info('Unable to load machine list from cluster-configuration.'); - logger.info(err.stack); + logger.error('Unable to load machine list from cluster-configuration.'); } let paiConfigData = { machineList: paiMachineList, + debuggingReservationSeconds: Number(process.env.DEBUGGING_RESERVATION_SECONDS || '604800'), }; // define the schema for pai configuration const paiConfigSchema = Joi.object().keys({ machineList: Joi.array(), + debuggingReservationSeconds: Joi.number().integer().positive(), }).required(); diff --git a/src/rest-server/src/models/job.js b/src/rest-server/src/models/job.js index 1e24c38a55..78422c5a4e 100644 --- a/src/rest-server/src/models/job.js +++ b/src/rest-server/src/models/job.js @@ -446,6 +446,7 @@ class Job { 'paiMachineList': paiConfig.machineList, 'reqAzRDMA': data.jobEnvs && data.jobEnvs.paiAzRDMA === true ? true : false, 'isDebug': data.jobEnvs && data.jobEnvs.isDebug === true ? true : false, + 'debuggingReservationSeconds': paiConfig.debuggingReservationSeconds, }); return dockerContainerScript; } diff --git a/src/rest-server/src/templates/dockerContainerScript.mustache b/src/rest-server/src/templates/dockerContainerScript.mustache index e33748d52d..1d5073f56b 100644 --- a/src/rest-server/src/templates/dockerContainerScript.mustache +++ b/src/rest-server/src/templates/dockerContainerScript.mustache @@ -209,7 +209,7 @@ else echo "====== After debugging, please stop the job manually. ======" echo "=============================================================================" - sleep_time=604800 + sleep_time={{ debuggingReservationSeconds }} sleep_count=0 while [ $(( $(date +%s) - $(stat -c %Y /alive/yarn_$PAI_CONTAINER_ID) )) -lt 30 ] && \ diff --git a/src/rest-server/test/setup.js b/src/rest-server/test/setup.js index caf51e205b..c4d712e429 100644 --- a/src/rest-server/test/setup.js +++ b/src/rest-server/test/setup.js @@ -28,6 +28,7 @@ process.env.DEFAULT_PAI_ADMIN_PASSWORD = 'adminis'; process.env.YARN_URI = 'http://yarn.test.pai:8088'; process.env.K8S_APISERVER_URI = 'http://kubernetes.test.pai:8080'; process.env.AZ_RDMA = 'false'; +process.env.DEBUGGING_RESERVATION_SECONDS = '604800'; // module dependencies From df287ae0de64f2f59eb939176bbbc212892b8a8f Mon Sep 17 00:00:00 2001 From: YundongYe Date: Tue, 5 Mar 2019 16:48:44 +0800 Subject: [PATCH 4/6] Tutorial for user to use job debugging. (#2264) --- docs/job_debugging.md | 57 ++++++++++++++++++++ docs/job_tutorial.md | 1 + docs/pic/webportal-job-debugging-status.png | Bin 0 -> 10388 bytes docs/pic/webportal-job-debugging-stop.png | Bin 0 -> 15671 bytes docs/pic/webportal-job-debugging.png | Bin 0 -> 13028 bytes 5 files changed, 58 insertions(+) create mode 100644 docs/job_debugging.md create mode 100644 docs/pic/webportal-job-debugging-status.png create mode 100644 docs/pic/webportal-job-debugging-stop.png create mode 100644 docs/pic/webportal-job-debugging.png diff --git a/docs/job_debugging.md b/docs/job_debugging.md new file mode 100644 index 0000000000..3235b3fb83 --- /dev/null +++ b/docs/job_debugging.md @@ -0,0 +1,57 @@ +## Job Debugging + +When user submit the job, set the following property in the jobEnv of jobConfig. If the job's user command fails, the container will be kept for 1 week. And user could debug the container after ssh to it. After debugging, user should manually stop it to recycle the system resources. + +- 1 week is a default value. It could be configured by your cluster admin. + + +### Submit a debugging job + +#### ```Submit job through json file``` + +If you submit through a json file and you want to enable job debugging feature for this job, you should set following configuration in your job's json file. +```JSON +{ + "jobEnvs": { + "isDebug": true + } +} +``` + +#### ```Submit job through webportal``` + +If you submit through webportal and you want to enable job debugging feature for this job, you should set following configuration in ```jobEnvs```. + +![webportal_submit_job](./pic/webportal-job-debugging.png) + + +### Debugging your job, after job failure + + +If users' job is failed and the command exits with a none-zero code, the job's container will be reserved for job debugging. + +In Webportal, the job's status is running. ```TODO: show the debugging job with a specify tag in webportal.``` + +![webportal_reserved_job_status](./pic/webportal-job-debugging-status.png) + +You will find the log following in your job. + +```text +job has finished with exit code 2 +============================================================================= +====== The job container failed, so it will be reserved for 1 week ====== +====== After debugging, please stop the job manually. ====== +============================================================================= +``` + + +### Stop your job manually after debugging + +You should manually stop the reserved job. Or the occupied resource won't be free. + +![webportal_reserved_job_stop](./pic/webportal-job-debugging-stop.png) + + +### Job failed due to system error + +The job, which failed due to system error such as too high usage of file system, can't be reserved by OpenPAI. \ No newline at end of file diff --git a/docs/job_tutorial.md b/docs/job_tutorial.md index c471495f7a..73c31de6d0 100644 --- a/docs/job_tutorial.md +++ b/docs/job_tutorial.md @@ -124,6 +124,7 @@ Below please find the detailed explanation for each of the parameters in the con | `retryCount` | Integer, optional | Job retry count, no less than 0 | | `jobEnvs` | Object, optional | Job env parameters, key-value pairs, available in job container and **no substitution allowed** | | `jobEnvs.paiAzRDMA` | Boolean, optional | If you cluster is azure rdma capable, you could specify the parameter to make your container azure rdma capable. How to use azure rdma? Please follow this [job example](../examples/azure-rdma-inte-mpi-benchmark-with-horovod-image) | +| `jobEnvs.isDebug` | Boolean, optional | after this flag is set as ```true```, if user's command exits with a none-zero value, the failed container will be reserved for job debugging. [More detail](./job_debugging.md)| For more details on explanation, please refer to [frameworklauncher usermanual](../subprojects/frameworklauncher/yarn/doc/USERMANUAL.md). diff --git a/docs/pic/webportal-job-debugging-status.png b/docs/pic/webportal-job-debugging-status.png new file mode 100644 index 0000000000000000000000000000000000000000..de4bb5b8449f8641ce1585c9fb86daededc81ce0 GIT binary patch literal 10388 zcma)i2UL?yvoIFK2Ka)2bP)liOO-Aly>~(hD82U*LQ`HqI!Gt#Jm+B>x>ICQG;u|62pVM6PN>Q0( z-?xo<^j^#k>q5KINifxXL=`t9|FVF&DdTHZ&JpkDITDBEKr|CIosVLw?`Zy**5a+0 zJ5_zd_E6;K%ZcQ`x#VXk2?zLSZnhKw)js2c+1lDwVQ353!7pYDqer`a@^LFr9VQza z8+{`qj&p)ZbZKG7t?@lFybk@j12gdS`3_R=zNsAz39GmdCltxWH~1jz$i`+p$wcVD z7*h0|WyS`6IN2BGnBpE5ve*7(_A!rG0E7#VoL}I?hIl|;;N7BaXXY8`=erfm_95vl z36?_d{IcVj7C93zC#OW7%QF_Gn3BKfCHK;pq%hJwzuZBq0$X<9@XjYh@1svnhbt2yXr5qhLkaJj~j7S;aeYvW~}|GJ0=EI+X3w^`@r8 z(XAyppi%;~d)d+aV;M@I*H0!`{zYW^kQY+%LSUZJw?D|_&-HUpgO5bpCu}jaZ%oA5 ze$bj3iPv%bc3o3?*xv4Y*8{iAwG;8AAw)h{>(p_&%{{rS`mACxY=o5l2go>-Xa$FU zrW~AM?GpBQW*>xpJnSg+xojd#preLtM=y+lYkoo?WAm7cixzdaeI6VZ`Ljw`$RPx^fu4(6eU*z-KeGzsecB ztf!pHmY4h-v`Wn%agL8Zm1&;38&dLJudm9hV(U1ZDVnd6h8CZwdzCgzEnaB>? zU$5E3KF=3=R5EloSv#yNfo(*+C^(w8QS&8;VPNgf1=Z?5p!D8j%_)$FO0@+N2C&}K zpmx-u+>91gI8~vmF(;e{DYoL={;mHPsf~3}Jg1^0Ta4N$*Ix3a^U1p7ZOfO2ssC_G z3~Sh-ig^E{c$7s58W61@$&t!OUZHVB&D4SkZ5I7W3C{G`|~5B{(s!cY{fU=VDYL?`^e;j$Xj{hIB)MD#Exq z6tb5qT1}#Pxi0?NNK1sj+w&G7X<2r%RwY@ojoAE|QC#M<#9(*c=rN61%c<^ZQfRLG zsX|UeL0?RDv|%s(>*ZVAh$=)u&P=j}UGW$1JKC9nW$jTqx%w6(lVTMkq7`>H#6`dS zmR1syG`gkxO-dbO20CHjlA2k|4k)+L;lwP;BcCk<`c^Cm*je@&`bJ}mDLzc9bvWbI zCHi}_{3D|bn#@ZOt_P2bu|6ffOrko>qAmdvAp2+jwY5)NxFZd;sl~q_G&2K7Y>8=n zHf3!YRU}I51fX3`+wN=cG=*o%9M(II8T%O`(m8}yUUqnQ_Ut6wUT58{t-fSwSCvk zkPGQ@l0R26eTD;^;Vdb#Mrxf6Mid*kAXPIHC|ySxLV^;2kl$l)Em+C&`Dsc#8}{aR z1KPba>-aQgIfh)TU9jZRnJutOV3@E`j`#)E52~x;pb|73xVtSbBcRbI>EZljCiZmS zKNWQYMm{&}J>69S;{bjsvXY!`+at(+0+>#SgAw4n+_tviy<_Q`O4*6Ib%4aZuA$??M{qe+ocvJ@!U}_KC&z#+| z%zi9(!h~|P*rDnvzqT=DvsQxb+({I^Z$l@PG%S)$HovrUn&nMHWpr^7g{zfv>x-$( z^RQr&m3yh)_r!?pxBkszlUA~XtliDqEc4dWocs?`0_I6!nCO$Q2X@FEo{3XX|v^F{9D19o9U>&3VhD$c0<>mYC~6M z+;Fe|scx?HTYwXakJ%qskJUR=3$%QNR7Yo(eOvQZMYLa} z<%weTLxy?N7SRAZEg>LakG544S`@V?Np6rIAg)HfCS^BLDsPfjE#f-r*v$l{SQj1Q zKeLd~@Tg9mPZW=0r51kzr!t!{m_ z(>~5en@&OH|~b5$Kon**&9fQD~&UWW-{TV26{xyEZKQSMQaev%7eBe!J9OD zmYEoF=Q-{NC-D72S?2f{pg|tf0?iFi`wfr*e?Lt!z2tU(PLBC&1;a-FR0v4HDqj1N z1={Yaz=yau`oFK;=a5!)hRLy9=W%CluI=e8b`SEcvj|4@I?~g3w-*w=yflxvJXn7A zNorj$D5KU`l-51YthH<1T~y;xyWl1iMB@fKpoM1}Zh_ED&%um=oSRGs1wL_;+3F5; zCTVn6J-gZaBMaksjcX2K9SDLfLc4dbTn7)3k?Rc9DZP#^zj_oKW_GBj2E@R<}EH zi%r21H#{k%OctNrsU9>&ahkxxt6 zukCA)KS`K@26M*^EvU{&aVnUw6lLd> zmna0f@gm?BgG8H7fB+7)a)W9jj$%e*Num4l#bP*Lndni`%l0$o05(hcC)KARB=d_R zIWMxhiN!zeFgVkneNlfevf6xrAIK2DCWW8<1#Ng7K=Rq83`0(zK2&!V8znVlaOwG> zc8vI8lhrGRCLp}ug9g)FL)a`I&L=Qa=iOfd=rb#j)7KOwbOE_w$1~R!4sgJPNj{^q zgS|Fu@9~Dp*`h>lWpS-RnC<;zNz=rM{d8X6po~a9@XA z>{)_YORTC~UA?m7b*V7PbV+n9Z)Vy(THQ?4} zvNFD-u`QKnPBD!>bmp*di5rIi#8xH8u44UuT zT9qO@qYq-~l6!n+k%OilASTAZ2cER|@fww@GC+z7Ul%1{Eg8K>3J9bAHbKs|leI^^ zY-VjHM4`iF^b^S^L{i(G*%zm-R-J{Fu&0{axw&rX-#)OAy(AUc<5j3ildU zj8hshDs{E5JU5S(H(3MYNO*QoUH+c5 z&aI!LJ`%qs12h8@|90$VP*++@wQe8;&qM`^f^fh)(s1>uTIJ|+ed>2QMJ}qz6idkB z2%bY&+zlj$@eZaxol6EpEY-BOiglS2n!6(-i|*9z9jNr!x)~O|c9LKQyQw3mG$v(J z5uN$~IS-i=uI{OwIx@%SW8hpFN!7Am@d*iPMn>s?i#v}bJ%Y$wD6J>PLye|!V%I5) z+q)gl-_a+1)36}O@=WKq?Z(A^Vkp2Z8kBi6nC#LVl6(Cr_2_TY`2qn+_^jWi2a~e^I5ln4(F#xJ$AY(*VeRTBphg z;6iV*ujm zmTSCL78ZPAgf9+uA94XZ!az%HbB=Ocp?0)>OeDsaM+)RgkSpln4`E_ic$(T8n z1e+Z1fd^9mJ0jHYSx*;1lwMRP4qc94E&c19r{Uhh-ky+}*?D&Mb&&hHlkFE-V_KA< zO)umUC;TuE?k`EU$W`6ik%T#=|S6WQafgqttl&hnVm@}u@DEaE#$2A zNgWq$Pyc!ulD(rYawiTIML2as`b^g#l0B9`q7lGtpDKi6qKkdc%&nE3#$JK(uGD!D zg#Q6L@O95w1=W;rIhE0-msW#RVeY|biU1AV=AhBGYf@7y0m9Y7wz06xOFpus zewra}ww8?A4`*^_%%JT=hP^XjSlAl%6I%Bh+g{PbO_p%uXMEo@F$)nxRX00gs~1`4 zmmVywdxa_@+a<6&3IwI@Fx^exC@d_sn`oW8Wj#OE*Wo$<;@YO}Jv_!#KqN)E4cV1uiMmtw|Puleb( zZ;;jZjs6;`svbFQ89cqLZ^BMSU9fqqU5^mCPY}$#p~Il0sm%>jy z5D5%39+ho-FyI;nn@_}b4M^?69e%iB{HFn7)Y>H$PNt>|?b|cC96>v`zN1FK1i0K# z+v8TN0NT3KXG;}whp?`PyF)hcpxx7Xn8Ee#<-cmf-c*<9PM!o$@MJ#MX3=pfjdxNe zOWZ2+%cFAzaFR1BbxmIT{Zl}-pSwOP&pB=b~bG6WWMELnPZky*2x2!o5|g> zifhFG++Xs(#}DDBxu24d5`bQy3rQe{RV$L^?nWm-N}cin^}!KMu!asiFARJ|5 zRD71x%T>iuSGYbm>7F0><4Yx>d(!?$mnCeWD^rLGMb=E3l^nidJB<`sV%2NnoK($f zHP^j2Tsa8~Xr01EW*b`%9m=DRJyQj;R!rqID5_b_=Ym5cyAFP8J@A=y=&W_$qikB4 zvbV}z`vz#hsWpb6qhCz$G#;;92SRv~TzgFXN4>!8Co3lF(144Z^-+;sHAHFOJYA_g zGse8WgnLlPJcd>uFtLo{@+()u?(43-WHNee+lkkJQLzQ_B{q?xz)?byWMwH~ySL8Oq(`x9bEFVDHm--H6)J=FthG^}isG!Pj z&c=_ucRo{fsgk|%lC;xm-85~_Ls@f!mMBlpeohYPg9#@MeO(zjiUYZr-unxocKJ5y zk7e(#9Jf{F`KW0_FCT7tG2b}^^CWvBW-bd}(?d(s5TdK2|a%G(8>i z1tzlerX9Af*B`O%e;mqgSP|%ARrFa)k@E_*Skv!RWy#)^t?8+rpUb$x+a07$F%{+ADzVq@tz|FP9*jY2I91vD| za>D??+TyzP>}hy$l3($p##TT#!ry%6=#_va$lheIgKzZ~+SvL?o=j!zgdEPr2trO! zHP$SfFp;7sjDgvxN0oR%8^M;N3PY8&()MJ$5qP z@vTRq)Qk1t!K{I$<-qq{1DokKvML@Eb3LcI`g^nT^8J4$--yrebd16@bex@j4XmV# z8`P|7_E26oQLD`g*Vf)08oK2b=8{XFnkuA03n7UU^T?oir>4z1un5)V$&8%(kkr$) z*JJTEPiKDPXVC5x}- z1gb?3vZ(`uAmMdtmqL*Vb?Cx`aionM}YXM6*kZI+xWj2fpdJ-Dqz~d7TV1 zkWQG2j4-zlc>H6b$(Y~F(BKnv=T;;R`^s>rAjev3R#abDZ2S0R3yY10w`4%yBhaNp zcQj5#XA5(>HR~s|&;@p;e=CzD%bj#I!s4T)H|Zi(D(+-cCG;dQT=SuBaWn>7?1#eI z^>V3mTu+jKNaWYJly3|670ty--Ogsn84WotF$?I)+FrOYh;*Hyk-bu4L5p3acFN?8 z<R-2e8;%1GW_)zRc9Hg9PH)?7&gja`f(q26rK@I!@ws6zjBfN*~j*uOV8x~*Yck@1mB#lhy$kk>J^nxtr738z4Dh|TqcNDhw zHOKjk_$|`D5X_=fJ?u&VaDy##C^#iv!n?<2l}Zuxq|-l$1?LzG@(|M;bE z#Fkl>N4;5LN1Vo<19}_r7|Gbrkcwiah1WJAjE8dKZx&8<`iy@XbxReHskkh&otrc9 ze$?wP2-g5d00S-bopg@51#`rXsLhWQ-Uo!8IZ$6=HP|!ic7dTY#bcL-!%D|+r7=ny zLdSV5HDEX#Dq^zmX3OnKakPTZmH2U&f*b)(hAds&@Tc3j%w$w=+o4+V-m&covc6x2 zqRs(wv#o#3q_{5C-=l5pXbJg}%z?dOH%^MzJWy3h{7xD8Z`wAG&${yBpq)3j?QRq% zKq&Sg?bcL#Z(_B0rc#wj?e~IRrWd^@v@DX0@yx)%J2gMi$$B!DZ}$`BX&sg`R=g{u ze||2c3Oy~T%M?ZO3|vE~?k0+%$9HWJv~`*BW^{Yyb1Qfe*##acmL>M?vzERt;~b!9 zL_|bP#`nuY4#3o1X(CXtE>8SzrcBIP^%d=P9Op;?^FgF|%ozx@Ua>o@fwJJ|O#Vj3 znTy#au=D%atx7YIG?Ut||-cs{B<@v5^Qi!?&dVQO~)QXI#FM6RGfT`HooTO)z zR`y!gLRwVz6C8f<;XoZU^~GkY)x#%F#%0}-QQyxKf99Q(Y85il#`$}>ni3PTW-n@aG_)eUl=FR8 ztG>boR2XP0v!rvZhL~8J{kDUVJK8fRHjwKfi z>N6nqVb}mAy)_ouQV-*~UmUbs8E!wyBtYLa92({Y=7ko5Eb89oGDAODO^D|Fz6yKiAA7XztDSAsU!nBVSELYP(p}T(dT$2k1Wi@bhXAWa{DR^gS3N z_W04{k!|_5srUG(Fz&ZgfouM}-uxMhiS~G#2?WiZEEGu2$hmy%xv+6{;&;co8+wCY zrKUunLn=gft0J9~0|j4q=nB+xVtmmv!JlQtvK8frX`epH``uVVdO3eyV26Kg}G()z2-@ zM`?BqA$6kr#{?y@&zcMcxkp|6Rk)T`ds7C51s~Zu6Rl)-U_eGcnxVl&dUI&9n;sea7ReU?Xd}Ru0qqDe+<2)vyhz+^N)S_mQgRho538*Qnb}o_Sty={{ zvZVmLbTY|o`pG!2lR zv)wy6iZ&6LERi`$z-1eF3<}Zs{(8;xM8--1KDf~Lz@?|B|Bz<@%Q@$b-d&Bf3VM-w zo>8-EUo$5(94_6$^2nJA*TarJJrNV*G%|92TU4chmz3aW-fA4xD+&7fRBl$R{0lRn zxxtCS@}a=d2ln%wCoxb%5t)rZer;^yx?F>b&C0+DF7j^Uwfo7RC#&P0QBB~xC@?#B z=OpR-o*T=;Xf__mTq%j$5lp%~RRNqibI{$-odzIFFR2S!u9tIyiBR>4q0 zJaM{+RgPPSKghyA^p^>Ixr=YB0QC|!zQ4Rd1Im-|QFiY=@yGENCgQMbBS-ZNiM%mh znwxTZh5_HvYf?q9!HU)Qxz2zpM4srwtfEf38I(n!Ag4`MGc`8Vgny_5aVP%5tl>^) zD1InnK~F*wsVn1a%RuQzT6p_JIbQ-_zpS|anQVP{dwc>A@x;!_GfDbrMRi$GIDT${ zK6MU}X}&8$DcU-q`L(SJgMTybGohJZHLj>k2(0h-%xk>ANCP0%&u?B&sIt`eP0Bcr z7j#M6k1>{Tp5BVGzi{aVr);~I$Gasak4g7+ZOaOyN}SRXOV*ukivrIMINJ+IfI-<6 z**Rs^t=ju#Uli>f39-38Ba;pg0qeI~Qz!Ync2;z+iamiTe~K-hJ(00BKKlwf{3H!~ zXd*H4tznBTb#6pIXQMdiQxmyI%0K`t<#3<)91opi$Bqryp$ib1!e=dt zJ$?t52qE^3hoy6h_OTRy!pB3Ny;st>{wXF|t@2IDfA#XR&>8Wp=fRoR%Sq{KrZXF_ z=3|^y$OmZ+*q&lWyL>hfr->k4p%nF34hjg?*guAwqh)41(nNvxRVaP-TX+u}fg2|Q zC$UY$zDy3qKS6$~@1bOmbkfF#KY^6y2MsQy>+@AxRJW(z=|=nVuz*gy@pB#eDdF|F zxazfCmZKWpEa4GLKsb`jR^R%v&<7@=NcKM|h#Ox2iQ?<*_YG1~kN%aUxa){wn3G7f z)|~(M#KOHx=YsGD!m0kjchUU6Mg2KMZTfFt#PdO7qLn8`#5~&nu3{-HwMXRhu?GMA zkalCZ^2Cr>y)VUQ67u~XBV+1$uA=dxc8-Lbj3D(4RQ#vXMmb-dZiD`wrEfIoZlx@~ z-jl1sDvC}#f z6l)l(03H;yyjPbn2hrJ!r~8frxDHSmf@@&0RH9zFS!v#E6aUnv@) p(uo7j%sqnvHOl|qrG^LCvrB|+$G*uGU literal 0 HcmV?d00001 diff --git a/docs/pic/webportal-job-debugging-stop.png b/docs/pic/webportal-job-debugging-stop.png new file mode 100644 index 0000000000000000000000000000000000000000..c88b1d0dcbcf622ea1141159dd683677c7949533 GIT binary patch literal 15671 zcmd_R2UJttw=W8!fFh!z(u*L{dkHNRX-e+gEG)uz zZ>80-uy9>&uiNl(Z_jT7R{Czgu-(<=-e6UZQ109o?pjMKNn&AD#}HhZ-McN{cX_Ml zj)nE;$Da@Ob0+yfEG*HXchZuYAB^{x0%D&}r5zrHPL^?0buiI?X^J;{74}Zmn0u4( zOW;#>b(dUCO1!U-|nQ>)AtW%D)SNK`#UUsin$A4^lqO8dn*~6ryKT z$&Np}MglK_L682ay~#!IVtBZDvUBt5!#}r;S-cKJn%tmFCT@&7eg63wYuK_-6^=lp z9GP?0IAj0)jNG&#{C}FqdWQF3%5wM5@cip|5Q6oO3=-XsyZ(6?6X_knzbj$glfuIL zyO3-8-|O)I4;qR5V;F^ZFq;9@j6I!s|Sk|kxm~J1HuL76FOHoXnO~D)lZD3d-by78wG&r>a#nv6hKBvZ* z>-OV4=Y0b#tY;x2Dg{#mi_Jp-20OUgAKd~z)!N7H$P`b$$=JG;j8EknWy-g=;|BNALp*0>M^bBA z_Hg*P7)OBdhQE%`G>eF(9Q1hz!T%*CRogob;$P#ez5%5Bqw)4%!

I1%pYu#>txf z*5^M!qnDY_s7wXN3B}B^bz3wc)w<%kQPmjz;H8=fZ0Cd!fa`AFnBRn2(&*^2bs@0(olaeh!pu_S}35^L+^gI|oJ-5cvC&UB4*>$&nATwVq|8#^z|QoikTF~#@$;S>y@94ib7gxe@DICO1aa024BeXpW*QHwh@!Q}q!& zIEJU2m6JEg9Nap0`cb-rtw`T*Gu zF0UgL^lC}BJp)>b6M(>b28|4^tD;KU0XhNKxr_ShW;j^6!U@*1#@*tJ_8hN40d2fZ zW4a&Af9(h!oP%6A3{agqpW7pXk%l8BqJ_ns$kDLMP&e=F=?YlCnZ}l)ZPysYb880s z%g8`?&)rWaH44+H*eg{)JR`hw_l>i{?>lUZR+ho$Od$iO-;8JaoEm}jdCIn+(#RFz zzRe^&=cMH7#=&3#qSQWbowD?d2~R=SYU-)2>rpCZ2zS+PHzvAuSsgEGct+B^=IvR= zW`85LTGdR`z}6i79G0fFpS?>aUY{X<8@w|nnbq7Qua-jS@s*OkT!<9xKKZkRnXHWw zy`$qhNkCg!qq|##nD2z$R(ATRfI0h{2k7xB{)v(2LJQBnjRxash)tpWs^o@ypY;PHof&lxKf_>M8p2zyCo+jy@+1&L@Z^}Uxt)fiu#*iZlEjr4KAa*9uR*e_prCd!m0gaTWI2k2CM=Y%3@kEMX zP*C@`W9ZbJ{x}f1cVj{sVZ_QD$)fHsB(p%cJ zXa@&a$Ax4e%GACN<9j|`hC{uy-Dln`^VuEfhDQScEUeCQc7ytpl^ocarcQ{xi`X@i zI~$M7yXShJpI0)5R0wKO1`Sl-!LrFOrXLOh9 z5>7BjQGuGZlw3`5xL*2pKAFi-9|z=AS}NO-cZQ02Ys`P&%8&P{H6V#>JNwS2xc>De z>-2phi3)!x6(6F(ID7fZ*ezO-cZH`nOJrQviO<|)KL_!8C#vPt8&Y9{u@WS|Lbk(2 zS10GKKDk{Z2XDv%uJl^3;MLyiCN!7s=;a&;59U(0Rj~$kfiPKi>Zm=10AvD4d>3Ek zEEnEbwfbF#thArQ5;>rvd(PHiys=%uJ90)VH?uEjI)%!OnH`ChG;228c-`|wn}t|R zK_VT`r2+finZ>joaRatIu5CUzq;Cuv1i62GqHtrb;NhOe-*kEdC+d!j&RTb|!kDH% z$C9A`3@x7XnS!7W%CiZ+H;n`1Y_4}c#7z3>TUTBg&Xy&G+pC$GicEgq;|F|VswOr%7zH@hZXQ)0My;DH90Skl15W)F zwal8EZ|6fsNH!w4MndKD<{gJs#H*V^^*_pZ1D0LcehYpNBA{A|`E~*7H{o04%Jp?i z8X^AB2fltRg`xs^n|9PtFLNVu!%BJ6K$#-$wBHO@8(T zNu-w&`!lG`C}iR>R-KIcUcSz4c$C)4Xfq}?zI4cIA^W3z&@d+1=j_)6sBdQNc7L&| zf7w2Cn3-*;)6P+_)3gTn*&2}LE|?lV@s9%Wgi1==Yrb2qx%l=5F}LoP{*oQmSB1oS z_=$t)8-}!U%?XREki%o_AImlqZfXxt&br;)N`oi{Yq@ zT5bA~vjj^sHv<*D=Vh$Qt?;@5w7#9RIo5c_`-mkfQ$P5@h9 zqS2ODvk`j+8z>a&Kgx*!pOB;3+?Gd;t4Gs|ig(^1wW$69Sf<50S%+q5wh6a&l2bk8 zAF0I(RsTT>|4sPyKS}V9jyT(i#T;^El=uCWuMb1TXLW)RI5g*hk?B4W;ODkN#{|d# z6_uDa^g*kVlKG3QQv!}AiQY&ubC_iNlYc~f=xd9e>F$4Ab6=V-;?Yps&5`|n*_THv z+bhFpM5lAmt6|04d%dU(g|{-@@qhE-YZU(tA4lYeA4Mjj!BmvYc89XiorxXjjuUN$P6G7*;qmW1nz(|LNVP7;=0&8W{kB&D$?tg!<3ZC<|W9nUJzIJL5_ymMa zn!fw2ONe9PpiQ~5wesXhRSSD4_*iu+K!cO;dS8b+9)7S`*f1$^Z=pqK0l(+hQ4yk) zPW-a(OltQn*_U%ytBIrjuH;l5Pq<;$N4D0uojZY`9k5yBkl#t{$^6VlR;)EO=Ht*+ zy8m9v%wB7~5F5p$n?M_R8_&-9!RpGrv1do{?AB~$p&?7hAqg$B5(>bt;=c@B+ z;FF(gj{~Wub({SH>#N=dfIjzJ7lzA1-D?%p%94ToJ)y*lse=q zVKMBLak@XzP|^Ue3yxt+8b0;0#M}6~5$3Fu@dAU{Fmel!`@W2(aZJG5EC8r(DH#lS zmMx_m1uj0)%Xsp|;Iavft;xQTN}3yPiI;w`LVw+C-nw`LDcGUHQpNAFTvwWKF{diaW+nBm)SkW^=ZJA57x_HTfZSM>BY|vg_}>#5 zN77cjuZF1f^>Q#rhw)C142eZXiuK94)XP`M0*LRpm;xoZUv-wiMFEv0iar^QpW9PoFaLGylzxCR^@t|OZYi@J$gpsx0q z^W@ujx95y3soRG_o-vISY}8(!4g179r9 z7m{6>w6r6yfT|?YN%tQYa!+b{nC>{M+mRC z3-M&9^;=0)H9iBB^{+7AbP*}CS;8FGi#&Q)6OVkxGl`~n1RRse>^mNN6@~>Ij38U$ zT&md~EkB@wHAVP(BrL+KkX9Sm!1xI;=_FGL)5K(JkK7fiH7DCI)zrBND{pJc{Jh z*17(MCH6W)39sPX9BGJl1rBEwCIbzquRnpKUXNj?^G{r)z!HnHL4%?l7b#gauV2vG z3Q*-<#Md=e5JJx2COdx@Y(Lwb$G#-h1G8#VR~kphknj0Qs<99K*$I6YfizL!Jwt7d4gc17^*X;(?1jfnd^+yb0GBiu4&dQv&RSt4i&SAA)745Snb-;Syao1}t=_ZfVm3{M7v4uKm za0frDQO3r{9>|_EUO=lrfn>g2+D3SCo`d$zEcdv-e^LAShb!zI{oUjujyFFXHEk*E zrX2Eke4a_*Y_BL5d8o7tG;tYC(k(4vPI=kS4V_#`DrW|FiL)fXZ|=e;&p4BIo#e+SJ}r^WjChbJ`PXZno5<{)-3<==de>k6+V4NOM$?~DgHz-%d!0q=w1GPuz%D}Ou_yo zu*N(`gl6Ra2-16`x{6qAQd=JeDp6n;;`9_VuB?R0Cs$ResaGl~N?0Z=0wEozNeU5_ z0U>!EF$y!BGFFOL=13No>N;>SpG$If!$+G03a5i`yl{^!W5+EaX7tnI;zjPSRR&(e z_jg+^FBQDqGG-VchD&Fc!^gUIOO0>l)K1Be+eG#oB|HJpCU+eS+flI$IG zMYoq&x7v>=@I24bZbsVuaNmtr|sWx z80{x(X9~cdDLxa^+$W9(`Ls%D!Kbn-ADBT%Do^nRerHRb56FF=+dGpzlFQ1-s_6jR~yqt&_os5rq@Vl8M|f8dN>yY3j*wO@JMuTSD#K#!e5eAp*3yU#w02kdC8VI zYL`ET5d4K4o5}3=vp3yUJKD9|>X>%lSVH3?<@NUHaHWFo8_EK4lvJd=xvOGufDCE& zcV~Q#oIM$U{*;NH7|Q1EDy-x@WSP9uV^MWYf1^Ac#_6LXnuKA6$1*mejUvxaIPq`~f=PYv@rRWXjy)Bly8X zrIx&)gMkh{SAX@Kt5Vh{4r8e6hT`SNdBMcI`TIO*p05_`VZsT$ofLO@G9$zR;(}F0 zTwANUp%nFxGDLj-79qvpL{OgH?MB~#INR&hdd&3Bp7xD8?-Gp zpWtxO?iH-#QVT1+rw{)Xwu^?I!?yxS!U?0!2kF}ZV8Iqz zz6+G1 zPh(h4h2FIsO!BjIcbsT2OD=cRw5PtOnz5V>rOT5IHz}|qc3*?+;v11|O=S_j@+cb;?3kKWnq~y5JAF zX;&ww4-8whElOZ>6?dpsKKLt_*UAYwO{&@997{msH7)1swj1|*Mk*gu{L0-4IdLJc zpTH2TYn`U{$38UNR8#jFYrx{0+&$VV3xFl$3CRxD4T=eP^tZZoEZPK%xZWsn9Kh8BZU}rXfc51>I}+o47Dx*ex?OJAt#isd+t9Q#3N$83 zPEuFQcV{EDZH|G*c~umpGsus${FPJ%8KG%C>`$ustrFxm7x~))JU%}3NkpbK7H?i%)SSN_0DLr-wVq?qsN*;5=Dl|{hMxR4!I|6oU zHh`TI`OJt)85z~mwR9??AM7YneRNY?_FEF;?;Y5v_U}koKgOM+F&2+^XB!;{FeLv} z;Ls@vPb9c2gI&(~u2ui-Ru!DvAAicm535fR@a!ZV&_2mJQk&Vt#m(5TS8|VTS<^8( zK%I}C@+lBCY?wdA3y({^W@`BoYZ_j1_@1@^oA0ccu8|G6RUzKVA95F1kl2wMAS^}# z+I1c4dSEz@NP9w(Oa@FA%F>C?C{^8pSn*{SQT0e(JXO>n;XAhGPh2-IGML1kPV_}; z&Sy(W)))@x2+PFZGa)gNzcb;20ME{mq9%pPn_4Hv}FBtLHGd)O2 zpkBcaiPZTGm{F0(wK%~zUOmZoM*(?In+r8{ig~VH`a!n8e{0Cy`^d`M(mq%> z^#*Te{eon}RJMz-I6v9^oAAb9G`=H&QoMlur;g|fD}9e=;jMVl)bA3!PY6SDb>=rt zonP63z4YrRo-nPgAG=J3zliJ)^BByie8@#_wrf6E=R*gB>8JyMr+apKS#5YDmMIEs zVa?F5MPaqrY-pyeY{IDNS*+-TCsizrt6%8c5_{To8)!9N#<8Qhin75ld^phCEnCJl6)3MfT+s%4cx)K3;#XJ&f2WjL1sy; zjHk|GCptz#a8%NuXx`e&8TC~B5uUu`=wa_^H#g-lVa122D-e}k&YNMP@1^F@oXXoh z+@Y{$p~J8_zSf~qIn7bGK>wI25W5K!UrvpR7QOxvwt}jbsyUDdI5f0vwhdwvL?xHI z=dfBO4(EmP1Pd<#4_X=5)8y7PiqisBy;VNkWX>3ui&9aHJsS`Xtc>{JDY*0%l5@qpFmpDw86ec-FICh$i<+dS=Cpw+YNJ+-EWbevlkZqQ15;clXF)8TjOA8+O1 zn`B!SmMyEapH^U>CPr9a@Re55!{<$njY*y*j@Y(n23pbeikqUm3HB|B<7NlI z%VNl9OTO&Vxw{xSv zokk3tFH%?w}6ilWjNO@dL-Q^t*C>{U&QS(PwR^<(?N$Z-lw71G^U5I zcCo634U;rBcl92$J()KPrRq%S0Dn3wSKT_^w{Q93-kdfffM_0D^e$v4FL^IK=M&Y3 zv)lAl1+r}TcaLdTn-@iHws^Uw;V3Hw29!@5jodeJ8P<%4lqV?K8TW^N< zP4*L?@53##nWa$08MhckhO1ycvGcO9Jl5+X{=ztP2wjTK7EpOjbrG-En{w?MUvA`# zNa8iU6a(&hw@zE+j~7iBfj{?)ApF+R-`X44S%0GZZ)lhPR2wE>VBxzfXm328$Mx~-yw-1EKD~O9rrtgdeEPs&djbgp7v-}TKD*r9r z5L{TdrQ@fM?HtYnQRG}d^MjP-ht5l0oX!J}Y-_M&Mz@qYZmDL8uRF$9>OG!Mu&`|6 za|U_k{Ar@Q=77F+hGV{bydQ8(x5O}E-s|_Lv;s!eLp?pPiwu0T($TF+IxRDn!A|ZrL+s5(t>RBA2M3PG5|S>w+?%B zHm}pucauI#cXP<7v0qk8ft}e)8weHbv3bD%Z_Ih|u#VGp@qMrDC#*dmjfvXDKqEjvIos%7pZ``CG|0e2hqYknwnaG5c4D!G+?ASAL{|fN)ae5aJ2(*ql zgQeUO>$$O#PqR+FVPKsRp)E=>UaLqj2}=5(getDRf_pBvM15sRo}q`S)#Q-D-XD_P zM)*r!@GBY0usmpGu)0?th3jsdD{4 zZ2A8S*>(T-iEsbdWMJ?6e`Xif|CAH?ze`hOiMyEeN8G zsV@Q9zI`2I9Enk9hKxFMIym0C0)7=54ZB5o9I*>3?W_TsDUKghsap=XZQ&nuH%;Ed z1`Bf03kRV)8dbNcDddi@@=#MW9(+~2k!I%AY{*hh4Wxu0v7r?&H<~mRby@LpqsgJw zz0!kyiPPJ}99AkbaGHd@DU~%E{T3d07RN3kYm{E~u>6NT4D+!(G56tN4v{p1pw-eJ z@ySI9rSFu<-l;s7!mw-PpJEczTb{R>1$g@6;igd2gT9;&g-vSX*JwoMdsxXwW(8P$ z`)R?raO{2tQyE@~5H*u$tZM_X3nt@__F{e##bk`c+G()sDNidKdLe$3xuc zf{&njMw`GXh{-kcMMj}>58VNR_;@I0IqqHs$>1_hR>@9j3~mLqY2bJT81-=ZCvyA1 zKz8tw(Vv#WxE>}sJTVx!bfy?Tjl&5ULQprr?4p-RqWiTy75})*4z?>D9lJ=oEI4(b z&7_;wMZM^}&O%`}1>a#Py0W7?L!P{T|BkRKMMaqpTZ?8(4n=lf$0Mu}zK<>eSyf3> zjKY?id7=3;{?58^KXyylZr587SCvnH{7*V~6-v)&7DLmZnV6*Rt|AB%RPHogAL{ZqBPcuBf0h zXmv~0MYhpimzB+o3@;Sf)LB!Qe9<-tLU-$Z@_J z9lqYqqNOO%(DuU1fs3|$?_0<*Ca-Nc`$;mzvk%KadHFDU`Pqy!;bRuylj+TC=G}dq zi522uG~NXl3{}|CUCbJ|(moSlZL6H#pGZwUeY$IZ2Xw&$f0eV-91Jp1bY;RjqF=so zzn#x=-`(5r;(|V{ZwQu5xz){zG-?&?xwhJKcJLR|_iDa=OV;i<$t~)cFZ|k2Y!DFN zTTz52lyc5Mgbk5nlFMgo>urbRTZwN(FEQt*t{1fDhHaxKXEh`DLguV16`NJH;XJgw zum#88j=qfNR2c#9A(Ow+Wyj9R%s6FyxlaUNXX$NqOgP46rT8&h)(dSOQ$ya3bCVrF zA9w~Qu{aX6z^WGj)z0sOFtja?#(X*uv4=gD3E321A&uo{1&>gq)CL8a+f zKA;dfWt&_AX>s9GlsC5)%&2D z&6MK=Tp5YGjc@H;(X4-Ein(X0vrpd@zaTECd`kU+eUW1WF4GNzKcYWC3}5)uTK;*% z;`Yr9?INt`w~#&qi^~#YA7%$bRgyvRLAw=EHiJiB%CF03RBs!osYX%xTzw|)uK9u6 z=beAM%5ZYgAa)x7^z6&5p0W5f8{&UJ)Y- z(pJM_QwBt~Vhs*e&B?JxY2|Swef9zeiS8DdEn4y@IF24P2B(Ci6e(dzwmEv%Kl(8% zT1i1*P8igE`7@RIVh?E%J3^rOurl|d4OPO!u7du0mD;qDvB}d;&LMkZ%Osk+3TA~o z9+D~vuwZlc$~Qh+H-&>x1)iY1xI~_=J=n@DkEE2ip!u;R^(pCD^muYcC5v7W(--TQ zMq=>upzj-*thHj)`wPdmjXTJoFR#zD@ufc#LN@Lc>22K+NwRVa(A=NG)Tn0xhFY{( zide3!xMXs4D(C&!3A1{6md2N_DYwkoO^gTmy3B>*8=X|I+qC2nJ^p&0}>iRE-%=B=Q zJ&>61;6oHqp&9=&{tMx0#-*0NI#_Wz_38=5+DM9a=W3~EB%6bfPK&Qc&*u(_s*di4 z#;v;k;yAZ7uwyFhC@L|`Osd*#pg*%akxiIXZNA@LR_0;iT!`%s)hn4fSVP?SJrYB% zmvjBkofj$CiM4q0a{HPmc2(%DClc`AzRKS$dfJlW{=IVC1GupPDVpMr8ZB&e9m_C` zp&YXixd-ICe=UH zZH!7aiye)2Q;1G2(_Ciohb9_W2Gi|q7{wn{&^XXIUlzJ4Uc?E??sJRIszbBWJMMZ8 z?f3z~6PCTff`*=-L)RA{qKDHdR)?A2_R+Py(q{I5heQ2*LQ#gPR z*vL-C^M1=uI8MYHaq^%MlLkC66aQs5{67;i@O6m^mfTU%F0*(+m{%phQT+tJka@g) zFIe^ixVwUa+avGOHtt*S$K?~;1wTUO9wX$L_gI9`GA*)ZyLlfHrEDWQLrbpYYgrg- zC%oW0S7;57VNoIm;2W;=W>{I97fh=HFT0~-OKtg#jqX}XJht!%THcUco`&Y~hwd2V z*t7u%k-LI(Jh{0`T}>-;d|)kPgIYaq=BZlBV1o?klN8^S##%v2vzpWgszGMphSB>A zJ!lVq{3E}gqkU?6B)FBMFItFu!c3*#ae?(^$6R7hv&f?IQXGq@Ci~QyZBI`;JO?~3 zG@@F%lmsm^vVTDRlOx1l5JOZlUi|gaQQ)^?d#paCsaoXJd7Ug2x%JC_Hq9Ad!a` zThLeSV}w<$C<+U_AM^VJw5o|SL?sqV_I0Yr^UNk^-rQd9-UF$rjrjhtBP`9l4s1%OOKuTlJ5t47 z>xWX=Dojvte73x(ecNAQI>(JLyrp}MN-x0F!Ya6;S@0}~rN!HScEu)`a6GC)rJolD zJqtUOi~67uv~wbw;;BONYq^KP2Wd{#Hr~Dze6bRIF@vGlYw>+74W*c3vH2NIjc&V7M!i;7KVW?i!fn0U06a~jXx(yf z7*0|_x-9_bzL2gbaXByGHo43P~NQbs7>z59gXojI@08UM_u|Wp%eq z_|Z(6{@Z~IwGu=JYswArmQQ`VVqkM`$z2@(?{C?~1yBZ`L23in8a&Zje~no9w-Id0 z1+LudDvyuUi-tx3{}x++VHu|*&}xQnTJcQ}?mVro>ZvigS#T-fS4cqZ>c*c;6sP60 z6idwZyXuV9UEGu@cDWxX+Zk8M6p~*N>@~Nb%bDsMv)2#QSi7<_8JUvx$)c@Zd$AbQ z(-7rTFZR2T$XBn0aVL&(hdzY7zy~ctib_DWcH=X1+CYWXTg=h4%L)G7+t5`$haoyv1m-gaxeFriH66YfoI&y`(koJ%^OYccz^i0yn)I`Wq%QW&qjD@E7&{mAZN)O7$DZ zUYdZB$E1U5fK;)122tl>aauATme0T(tDF?pJrRnwj2xjLCJvDsybEAU zm6E)WM$-E6Ep(>?W|=)>Vm0BMK&yaZoj0TIJYOvD|78PH6#t2&AdpHv*uVa1s~2C! z{X19GW66AC`sn)*r#EunOd=*yk6ACup(WIXF7?w(2}C1x8ZbAcajj;5qh99`BI;^*O$El_PQ;mj@m8)zV0o%)wf}1m z`dxy?r+k>aKKaUNx9YVr^Sy^Jz&DSMg-2d6Kt;8h5V}CVHNZ#guf~6&7*c=ym4|u* zCDMaYJlG)w;F~8R`-DDJ))DI-*1_C%-^n`fKg37T{pnxVVb3$WzdHKobng~A|6TY$ m!~y;9%K83ZCr`hAxeD+zQ{MU+A-MlX3GZZ-r7Pc<2K^6ft~B5P literal 0 HcmV?d00001 diff --git a/docs/pic/webportal-job-debugging.png b/docs/pic/webportal-job-debugging.png new file mode 100644 index 0000000000000000000000000000000000000000..018f42e105977ff97b1d003ae9f4b708f11b8fb2 GIT binary patch literal 13028 zcmd6OXH=6*)Gi>P;sI1ZrK2dl_YNw(6FNvyq=Y~~dasI#H0ixahft)~0HX97Na#UD zKnNtXfDnNj#d8j4t^4Eqao4)vpLb>L_nnzNGkf;l&$B15b+uK<$>_)k2nfj4R22aP z1Q+P=zuPX6;-9CR(=r4EECgzb_YHgqHy0?K5)ZOgw<&MCC6mvof-i|yi!wZO_y$p$ zSOM#TF9N2Y*G`}c44R-3oqW-ywguqUuKhBe-Wr3|&_}i0kDCRRR37+Qs%U&Qv=KL` z{xtFC7AI%M?xz930~`CxmCv34sZR(|_D2ASAx3@b`o4t}CjdalMELt|WSiC4_+J?}x&J5CzuzT}1x~ z@UN)VLlp9L<0r%a{fvOc_%Ym0YPy%5eKupWl*>t===Q*JzY(p1)m^eTG zF}-1#&MDx>mzOPm*fPT)^nA1aeb4SGNg<(lcjoP1-Q5^k&9t0~aC7~;I{{(z6ZVSG@q)o8KWV9+-m;?FN)=0MSl+QAN?1Hbdn4qCX`5eZRYqOb+|JIt?az9)D+-TE z@QZ~qjhh|PTGFKL-}36K1?;-_rwRh3$w;qOZpzl z{*!gL%bJR_w{L(bKTev6T5>l0QSJA5od#qA?*x-tCZqohj~}*^ho(>Jot{~{&inN= z7AnB*cwm{S0M+aR`hP9(s)Dr)p|VI1)e~G4&8;<{(Onn>`9`?mwYYXS-+r0obv<_- z^djonl1Fy-ALHp%B(5Q)vx`xdU@&S5WNAm2+t7V?o*1dPRv~Qr0YyF2qVL>0x(W%WS&jF_DEZ`^=a;$U2cUvGhiy=Plsu~^_tohZbUp02KVx2~tn zxm_x^;pNaiY5Naw#EqYG&o)IigT8ZSLF0LZ_svrF1Ew|;N`uS28V($+MNA_KiO3#p zz$q!R-+CxBVu|TV63I&(728j1A13d2ITO6;0w_-!h14z_&6IaJ zH}9>heY%^Z$TZo0ugK6?WDKhI7Spiy1Jun`TB7@DmCL|u8S*4cPgG_$_vT`nCTe&7 z;OncJpcNV9z6_!HI_v_I=&`B6cCGVz6x?Czf)_j2iGK6|#j)yz-mi2qGIyxF?gsO5o?M}_OI&+@ zF|*jUeJ=k()OUfDdkQmOm-etz zYR`@_0zEdv9pu=q@Y)Rja8xFBO z@Qe4*E;`neZEvzNGffI|E~IVVtU^S!AT*_=Q|<=ILr|OzQrk#p(W*bkS->~6O+}>g z=k66p;WUGsQg{$1x)nJ1Q(d+pxQZ!_-NJV{*YYSOgp_o{*zeT4?qF`ECs1{CKRcP7 zwZD|5@;j;Ya=-d=4WN<;d9l#PRP;F<_Vu#RP|GJ#M@S=U`gGOHc=F?U$G~HEJKZJw z$l@ZM=>0W&dGWxiaW3&&ey3rOk2PauPvagtW=LBdfSo5E_OfSa1CCmsxltPm32u7BG<&Jf5rWh5K%}4y$w*uyRl^e|yRGA6-MBAkJEBKZ zFyFwVZFk`#GjkIvLvL$cv{$6@pAeT?qJRwTq2In)p^*D-|FNN2=|j$!**9U`8AB4C z)qa-J88vzXbykPRlqk!DSCjr`YyO3X35$`qsqe@+nD=4$VNQdQG2uBzNXaH`Y=$vmCUBrjRQ+%8lg=wfm2cHpW>BZX&x@HAJu8kME6 z+kG>zeZ#cpk1%qVxGxM(+V75%W1myXXh|Q1|AoutW;%_43Unr&0 zuX0k&rxoQulkN^C#680wfC%(4CG9?L2#|I^Y=ZezJv<*!h|z0E2x!~1AA|v#G?z36 zHO*li6xAEhi-x6y51qh=2<-Vu|u+`~J@Wh0&8`Hrev) zNk-4>js`({TY--ac_<=#gkV}e-5-+RWmbnIyg+0Tg)nI9r%9J`j!~whjn+3$ggWY= z1ZVy!L4t`G{jccsjg~M%C zE)5f3GpQSeFM0zH(O}PvM3Ke)r)Fl}la-^yY;!?PcYOmdGLB|QEpLXwZ}=w;9&J|h zq{Bgl6KQqb6y=P1+@d4In7PUKO{$}{jf*^I!Bjwr$T!yE8!QHIc`9TcYu$Sy9Nu?X zQX(sj(>?{iBd1h7N=fmU&FVV_Kav-g1SC;%@0q%EtJj04cBSgdjlPVgo8B@O@e(V# z8C{eC!#f{+I(5r zB*MJ-Z)Lb`e%I0{ND0e5Qi6eBTf+t$yj;gFDCXJ|eUdE;I_%AU%ZAyO zxv4PwZXtPnA-h=v(QFWLk=&pZn!&;v{g&p;z4;LDFpO_)q}gu6aF{uB`CZ_mLqt6+ z@PmHxu}d?$t1sZii3HEFOi_v@BaAhNcWSF#zINrwb7-5CZCH~mbN&;H{QXK$q8t-q zW}+DGy>qXfntp8B;gK`Bv;l0UQssvInn#z|Ojkr3D?NB2>S@z;M7gNnHgBy%un=;4 zhfm2e8OH8`^P)c({*aC-lWWjw4Y~z5Xq?Yjf9Bl^B_X@^RDx6OBqnvTHuoY`Du7ig zSlP^SRlG(y<3!Z-jr_EE$-Tj+BtEQC>{)B|u-y2n^uYIE? zY>))q`9bFF-gt8c6JIRObLV%mC*VxvOG;O0)&fzCv>IQleF!jgrJLN6-ah~*E^Gq} z*(kDP<#_^;r#$TCjhLV6#ls)kS`CtqO3Hvss!R?+-|tuGYxY*`m0r)1vpb?jgdfCC zH2K<=HFfkH9SkQwZvT-|@5d#VtAFaP8Oy_DzxeG7do%S_C+fNAaTMeJM4`-eZl;_^ zEs4ctiAqcG3P*PoDBBS}Q(gsuXwQ#v9n)$cPFPR<*t%c!J}^90YCnQdmlfCFU3L)0 zRAk-wW;buTIr!sS4NpZ*pPXsgAW*--M*H&{hFX#|#K&zq_CQc0W1{wgXqrBT^Z8l! z7T z_+LZfcR!mn_Mmp9F3$i<}hl-LFYH~4upD(M3H3F?}hwPO!+qQsaus~?D zq*!cFJ**^>r|*_;yA>-p3cjGn?;Yfi6!%!6I*QUk2pwWMsRFYm>Rh+o)`omhuPqGo z9M_h8wkDeS_Sbv)?bQa=3UrSZ0;f>d`<9z-;^4vK1%%CuYEMXsg=Co|Q^A%duKSQ7Qlj0 z$p5x$Wtjh(IB24faWPmA%D*SJnE&3U*z%_dS96Dw=l$L2i`lS|3(pqC-6)x39d&x+ zMEDDmtA<|*bdx2bn-_@_aUu$=VF_ z@J-3CXb=@2_ns6$Q`v$tJkqVa&^64XD4B&)zdNk7-aO-OST3}jDGJ-(JqSv#Cmxhj=F#KPq3MveI5s8Qgkz*k6%yG3O-7 z|DM%6!~$MiUn^euJs-6<8Ag@{Udw-+71$f)mRM2~lmg9`Yf7lNS?fLb$*OF$MNoW^ zIR55qfwV$t$;9@dRh;-59Brue6Pg@KEX z$p|g1!hT=a10&4|m0zoufAEt4 z6j#aX<{ffUwOss-6dfr)*7;h-Qk*Q@$L__h4YUCaBT#!jffZ;ia-rYok#37%R+Bfm zs={>6rR=+lzK?&3OU`8asGNRz#mfl8nlCPJ+@}A;N~Xzjl_gBKO=XnfI8B=}6F%K* zSE*LYg~n0AoskgF*Hzj&6|z5$U4nM$c^`exVWEHIb~F_e74=;>=TKAoVz#RM(2|;W zGgZw^Y`K+TnYzq-j^b{l?u(+yIls*EL%D0aO-nD>g zdS7*Menv=Z(ZS+dyvUg{e;A(YM1-Nrg-p_Zk7DmdR-Yqx@Gkcod*{=wpKToH&^iH; z?`wy|rZ$BM3W$uy z8dfrN>7x7utF>|txaD=%z}ol<$Dg6E-nAwoM=OUtH8IvD42ueLi!rCY-_xW>{bw6O zfmd0^Qc0gugq~wNoT%_JXZy^PvnYNR%E^g{=_W#kZl7a3Ql0U#+Rdm5t8)uobU#GF zhAvK$_#A0gtA&>}WIa22b8Zw7{3dB$5PEseAgmiMUv(I25Y`SpXC|qn_@xIbtX%yw zrC;g{iQ#24UOj94=SG=e!LwZCeGEkZ5`&>LYBFCe!yuY8%{5=sa0-Z!y~q-GVImapP4@pk!ok;QiJulGiQf9 zOjqTfzIb*#*RJ*jDza9H+N=Y~xEZ`oNCF}ye^6?fHsZ=ea=2PN@5z0s*GSco%kcAu z=maeY?X!0R#){t|*lP$k-^&Yv0XiMD&z{X@NvS8mVqBZF(5tmYmM+p8Vg(Y=J@49suN5yZ?=R@b zVjdMh-_Is`Tsr!;(C`c=TcSN$|8_49AkW)~WS>oVk~f4c5YwZ>$P}129jpEzu@pwX z6`GPqi#kN}`2-omMiQjoePJ5(9d60dU!fVrXsy?8l#loi+cJA@WwU~s+j|dM(Xu(v zhpB>TiPy+a53!aH(b?>>9aE*Ni@Zy1MLs{~zF=~^&GD4uiKVUF6k2rY#4`QF@02YS z)W2ZYC{@UZkrX-xuIt4L3U^#UU{^6JmH7`j0c~eyVXb^gQ(#Qs)K{6>0#RwgQ)B`$ zA9fKhDA4=}$L>3N+RvlcI?fr8V-gH0N`_abD|ZmD*q0&q zAM!4YEVBA%U>@16Kb*xF?z?9>>A3ad#Jg~f{S-m0>*h-h+-xWY-;?ZAN{XX_tW9#; zb>Pl^_etwIN@b^-=k}S8S|VY);jrh;MOS6$AMP(~RhG^;Oy()K)Z9f;>Z=mHVLl3g z%cRp?Z4$Ft_pQ{ZzqpZM584c)tFPnmy#O{sI2#QU`RBlJM%>?!Kg`6tETn+TO&WBv zg;1Xggh4a5xWg~j@UY(Q#n(V7TkyMKEfpuLuM&Vl#AM+Hvss0qwsZjizVU`L*)?vX z;qag|aMIIteRuCui&~(+uFZ;_FQh3@&&pU6&^c@vi7~HgG<*Pjj~WKSvgIlFZIjoarBO*wP9?a@?}gj zgR#2Mh-AQpv=&k+yo%z^#Ud zUStW9!RUr$NA@w%IS9d&J6}7_u8nQ@TCe03Qn`6Bd(nPzk!{x-Fs$C7QRd`e>^!j8 zuNZ2``igTZaA zX-=fE-xSCs{8=eCQ-j>S0|RT4o6v($myL{r2xHPo6T?zGbwyn@2MG+d9Uh2F_M->a zOFJ%JYTa~`(~4>JEFh&v`;C^Xr> zt%^>@*L?6bH)}VouON?nKFoAmlo?BE*oA8X4Guhvq?{QK6ZLy<6HO|fnOMI9mji0O zp)qK&VL%6^S#6#q%daCD?D*0~lOK(CZ|Oc-kK)dhut)1T*3bMJuu?t+G*<9Y-I`|-RJaZcX zu3ucVK8v9+!?_*S*_g0VPjr$=F#D3l2qwe3BQCw&g=%pBqR{x}wHM~YRSvZ+?nf*v zdW3%pSVJ74mr3aK$HQkYAaw$FrNczIs*eju=>|j`U`0&B40cZ|W~i)F_czLG+(+nZ zHWHg?-viPe4IJTL$eU_57^*qtUpjR?n7tY{bSXN`Xa0K;o#}n`irFj!NXW`vBh#G8 z2D&9tx~LS;aF{x81!%1QM7e@C-Y9rD*d}%QS^5n&52p3mznWs9Ovs4~~zf&b-4j0$XwSF7y z)!AFnzPY7yON>gWUB^Y8dZgOAxYDCunTC=VTkJuyF^+xYb_d-lUr;yaz6%Vicq(O} zpoHMU3}?$e7JciM1Wf6~9!eRMjR*_Vo(YejaUn}&;D@u?PG^hIP-uA$4n%|fA;xWZ6V_!!VG)8Aj>j*1}m~$N7oZZNB|D0BJwP``J zW$vZ6H(&B@qM)$e>_W0k%FGG!8!T&k#FNi6RZ`z>4LF>ip! zMf88!i=zt5^cHxvB7IiqQ$-R29^N%Kih~ogR0Aob=vdhBtElcydv$p3-;Z zZGls%G4R-Z1vuT_sP8$XpDGl+GTrmAzgR;u=PRh6c%o?jq-FI3u;a3kdk*SxW$x?D zm+GQS&PEfY?951m1i@k3wxi9Hi`oZhpcIgWMh#t5cguT zjcwU%UnZ5s)J85)3oSZxZ?`%WEP5|^Dlxn0rlV;^7y09vVC5l9?CN6zC}~rm>tOmt z#in3=b@;7jKVM1$g6lcI?gD^}%_V&Kc8_pxcl>SH;%}~a&qgscfCT10r(H<|$88iH zD}+JoK(OQ144(e$XZiUj7J=2B<-w`Qyu25{r61xGFC?|!B4CR}Vxo3`iA2^Ji}=%# z4E>C-AK1mkS<_|o=KX;g*6GCHcC3-DW>$KUm}@F6t2L8&IV6@OTTrX;wy*(9AVtq# z&K$#kS0;g1@%t-EesT4xuv%w3H&!>#8#ED~vXAY^mzTn#Y9|e}IHQWhakD z7Tr|%ISg@$ML>-ZP|SAuALREtv$)FB$q;RQ6LAg@g>?S93BVTGTX_zbuw3O)$i24l z!R#C$x|;eNU#q^mHBIx|2LC+5@Kx|I3zKtzh=3^J7ZxLCYCS)SLN!6Ca`hF^IY6Xf zeI1Xx5;|@Et`7Z1P?v77j9v12%zJ(^n)u5PHrIVNBL9V51Vmj#;uotLrh@jyRgN}Y zg3CcumA2eHV&^qh=wy!*922%xOkO#bT=+*FeDuL@vujP$vk}JVP9AS=^?$ZUJf5Ib zUmP}5z<%JSI!BA6?5^@kl7E9UvxXu^%d&lQD@CS zMqt}%`lN8W28mZ1p11;?79royd#8`YW3tAO9g=x6Es6f9q8jwP`F*u>lMy1LW*IBN zLr6S!(MzJGOc;!4dPo1k5j9%qbS;s#Ik$1(Ca}vu#FGn^9k~x+|3{Bb2o*=Ce(}&N zT?}I#85LtMV-Xk#;Ro74*nrZmzDUqBknY=@l1xko#E9DxfV^W;J$<=7aCgeF?hSb@!GHu&g)2#_TeF&$e}Wlpi@$@ zO*CHY>oph8@V`(Bk>EbT7lUQ)3eMRI%30)&vTF=Z0RH@Sd2BM$4;=>e`B6>53`Wel ztL*JPV}Cn9OsbO!D`OvS=$88Vjg5SBv78DsbCFRNYHYr*F^qt|^z>0_j*)q$=pyCp zhxuU6#b4X2UXxNE9;IKtKi4Z6U@~kamG`|Ctd-v)a5roEv~JS6vG7pCWlgoJRhIGb zv7b)Jk`skl3X~o6uie`vD15mZTGEQLZ1c6ZGVI#gf}(`B{E@yeYh>7cL9=FmaR z+6PHtq0cc4^A*@wWN9N`s_GL1c1wvoY37CGTX|!fhGz9Yd9(Z$i0``F@jIRU1i>jO+}*zRZ80~ za5|qed4Tq_{MXmnH7ek}F@WGE1hyVWjd9X#nn12-7^-t-uWfF{1$Hmi%Wf1XN#0uu zr%N_iX{!jF)avJgmx-=)J|oX!PH*Ut^4m(kc&xICZ`x5blZu6hGDscgdi+&Fh(eY0>K-I?HVN#{2NdNnd)f=_zXZ z@b+mWr=R%I5hoDd@GamkZaE(fuEksu)~Yq$Bv3fU5(({^y3;?@F}7p4q-aG zB0ha**k~bUi+)+G!&c$g>keZ}cxP+)y#T;l18Q9HgP37}#etbSRes?tIxj}vc~f^w zx4gBCKX-Ai*3fz#Q2b3UdO!^rd9Y^CVBfy)&E-gX{Xu!XW2k`>22Ii}3#S5!vitHOAeDZg>a1^sanapFqny zo@G`i(4>qp6OQk4h%kO``_GvstWx-*A_B;fSY{f&%yQj|=P}!>m-%SKN6?+l`Y<;*)RHtQU@*3I9n(=GW<#y{> z?KaWLs&UVK{UT(5SQ(X-BjPbPrUs*Dl zfaG7Sw^RAerq{mL_s%$Ew5$H8)@l}Kh6wy%;9E-}@70(TV>^09kT8mNSU{(h1naZ*`w_GRK%kND*M9vXiUXXCx?yd;Y2fQ(b$Hov$mD1LJ4S_c zv0|V;PMekZx)Q^;YFgtyIydXoa2d@_0vQGPL5} zrd9{y%r*l6v$&5pB>lM$zndq_8dUPPgobdoE`>GttT?2$P)H|qV1gi~phm^|<(!Qr z$ig_J9v3UM!|9o(`?0_|X%1+recSS1S9LWt?yNR*W!BxvNwaamD`RCOi@xr#$0X%e zpjyzyK0Jp?o>c>A2!pCD&5ylGPFamJ_prI7)pFBED?N_v@2*1MQ8i%^=7vA6ML&7M zVa~{flTh004%F{iw9YDDf3{2{l=hkf0}koMD8XKyf5V0`x_g&xLi6aSh>NQrnw4>dz;2}{(pNH~?RxpHoa36(`<_kmfdQ_Lqkw579 z-HB@^vF8(c{o4Yp$|7tF@5jBd4MBxA(e$1ig*B}wBDN$h=S)z+dVmBRrO^GPMZ7CYR@HP+~h$B9jmXtOei<`1F3%J!J#O2U6Ifp#y2Ty z_Q83YxM3j!RTLmuIUu2J3Tpr+JExrA_MD)0_}yo(bLsd z4+o4gqrYoP_NkphPDTF$eSL4*%V$$LXc!_=RtiJizSOY+uk;po9PN4fV|z>+Ul}39 zz9;+B({GK7G_6OL2UDs@;BsPOMRVX*E%((C^4UxiZ?J=6u*X5oNB{qT25YY?LutY) zN(oY;tvWRA=PV<)34fL&Q32!Up;{y!FXeM+Q$_#le--}sefW9hLEX-2`=QtEvOJ3O zE}-fDFCW0eZ{-uNJ4Sx~4M!p#sl13UU^*{U#TVf2a+sW}zY&B!z{4bJ^?1MY_Q7%+ zkFj#C|Ff@T_Y`vVIMmkv&2aFb5RY?}!@~Z1DiiUtyt}1m)vG^S`3t^YCi~AuyZwU_=U;bxC);BIc{YwPgynN$$zp(=jFE_TocG2Qoi4eaKm*z7| z;}=hl7pGL10}CkSeke>;S_~RQVb5VbmJo&XUzA*;NLR#R?|=R#z!Dd$&@)*Yz?H}I z57hc~ct6vyjBPnq|9!Xp?EGKDyZKA3N$1^sh;fKQcX^l4`Nq_Wc=Ke5G6*~G1uO}P z$g4;P&ugXo3xb9T04|?z?1K+cZFGNb=lv!%LKN=f&j8s6)qCd~|F2gtj@sllQFH;R i#(AxEuZD6TouVqx)4d!~>-c}K5U43>D^@ literal 0 HcmV?d00001 From 5a86f3796b019a5f300562052092043d80b46be3 Mon Sep 17 00:00:00 2001 From: "yuye@microsoft.com" Date: Wed, 6 Mar 2019 09:54:32 +0800 Subject: [PATCH 5/6] Update logger type. --- src/rest-server/src/config/paiConfig.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rest-server/src/config/paiConfig.js b/src/rest-server/src/config/paiConfig.js index f8f1f3575a..8bed2c6b82 100644 --- a/src/rest-server/src/config/paiConfig.js +++ b/src/rest-server/src/config/paiConfig.js @@ -26,7 +26,8 @@ try { paiMachineList = yaml.safeLoad(fs.readFileSync('/pai-cluster-config/layout.yaml', 'utf8'))['machine-list']; } catch (err) { paiMachineList = []; - logger.error('Unable to load machine list from cluster-configuration.'); + logger.warn('Unable to load machine list from cluster-configuration.'); + logger.warn('The machine list will be initialized as an empty list.'); } let paiConfigData = { From ab49e4db6373f301055bf1d2357f3827d3ea3f60 Mon Sep 17 00:00:00 2001 From: "yuye@microsoft.com" Date: Wed, 6 Mar 2019 17:04:16 +0800 Subject: [PATCH 6/6] bug fix --- paictl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/paictl.py b/paictl.py index 3f58c20824..cae6a8630b 100755 --- a/paictl.py +++ b/paictl.py @@ -99,3 +99,4 @@ def main(args): setup_logging() main(sys.argv[1:]) +