From 31696a280873d0e2ff55c99a88652b330d0ddc60 Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Wed, 22 Dec 2021 18:04:19 +0530 Subject: [PATCH 1/5] Setup Initial MdBook, User documentation --- docs/.gitignore | 1 + docs/book.toml | 6 ++ docs/src/SUMMARY.md | 18 +++++ docs/src/assets/youki.png | Bin 0 -> 10693 bytes docs/src/developer/introduction.md | 1 + docs/src/user/basic_setup.md | 99 ++++++++++++++++++++++++++ docs/src/user/basic_usage.md | 110 +++++++++++++++++++++++++++++ docs/src/user/crates.md | 3 + docs/src/user/introduction.md | 13 ++++ docs/src/user/libcgroups.md | 66 +++++++++++++++++ docs/src/user/libcontainer.md | 35 +++++++++ docs/src/user/liboci_cli.md | 24 +++++++ docs/src/user/libseccomp.md | 3 + docs/src/youki.md | 39 ++++++++++ 14 files changed, 418 insertions(+) create mode 100644 docs/.gitignore create mode 100644 docs/book.toml create mode 100644 docs/src/SUMMARY.md create mode 100644 docs/src/assets/youki.png create mode 100644 docs/src/developer/introduction.md create mode 100644 docs/src/user/basic_setup.md create mode 100644 docs/src/user/basic_usage.md create mode 100644 docs/src/user/crates.md create mode 100644 docs/src/user/introduction.md create mode 100644 docs/src/user/libcgroups.md create mode 100644 docs/src/user/libcontainer.md create mode 100644 docs/src/user/liboci_cli.md create mode 100644 docs/src/user/libseccomp.md create mode 100644 docs/src/youki.md diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 000000000..7585238ef --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +book diff --git a/docs/book.toml b/docs/book.toml new file mode 100644 index 000000000..ab52e4c57 --- /dev/null +++ b/docs/book.toml @@ -0,0 +1,6 @@ +[book] +authors = ["Yashodhan Joshi"] +language = "en" +multilingual = false +src = "src" +title = "Youki User and Developer Documentation" diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md new file mode 100644 index 000000000..1642b6c5e --- /dev/null +++ b/docs/src/SUMMARY.md @@ -0,0 +1,18 @@ +# Summary + +[Youki](./youki.md) + +--- + +- [User Documentation](./user/introduction.md) + - [Basic Setup](./user/basic_setup.md) + - [Basic Usage](./user/basic_usage.md) + - [Crates provided](./user/crates.md) + - [libcgroups](./user/libcgroups.md) + - [libcontainer](./user/libcontainer.md) + - [liboci-cli](./user/liboci_cli.md) + - [libseccomp](./user/libseccomp.md) + +--- + +- [Developer Documentation](./developer/introduction.md) diff --git a/docs/src/assets/youki.png b/docs/src/assets/youki.png new file mode 100644 index 0000000000000000000000000000000000000000..40da2062b0d2b88f2603a753e66ac1058d9e440f GIT binary patch literal 10693 zcmY*S|ARcWgg ztyIMlv=JIazFq7kMJ%x-Q(Np?NDz_V$JVq!fO3br6HV-^Oz(W z*_q_d;=pHkWSy?Hrt+AgO_~rkWnxv`H3M-Gf;KZAOZUSkslUR1M2o#7l3Fc}4V)us=0&W8X^G#9`m`x_uIb->r_H@- z)-WF$J8%8^8mNw2dqW^3s(N4 z^Xmy!Sb7L0zWM*Adp&HQR~Y*Ncl25O{)ozJ_TK44b31q%sQPN$-i3_Jyg|LD7u9uO zhk<2UL;d6R@9w9)1C08JRq%J3$!8=FA#~5Be@Z~0<}qEWg|%f8gv8%@%fyP z#3mO6Y4{7&dXAW)h0?Y}d3Q1LS?l;ym30>pdtmiahF4)8)g>ML8)z3 zL)m!EGIyQn(!{@!#lN><9;MnoFTmZzOFYyYoHsopcES6rGY$k7Em{WXOZMycT&Z6P z=}Tv=FRm;QEw`qpB^$`K>8I^9b@ zfy-KIi|eh{ndXNf^ph0g-t4XK6k#$kI=|zC)-vn;Xj(w3xzy;X?A~sJfKj;@>X%H~ zc0jGcb-&nMpX)2@=5h!kU4Rd>2t-$RQbUR&GQX47qVMwaz8OX}F@+rX^ARZGqVdhi zS(_#GOWclc2Pr>CoJ18J%N`2d$0d+COcaYY&jCx4%iZAX&^n2S4MDtaNwLhY_A{Y-0~alORfi>`gcGWPR8KHl#;l;Zgwr*|YzLr0m2Z0?)q zDE)kvFr~SYbUCKT4;q7f>6J9n-Ac1rVgDfa!e}Y=hn-kZJ?R7Z!FXDC)oE$&W~aFz zU&1I_t#?*Zi5fqczv_ti6b!Yr--RCV)8TwPqP!z#&7Yp{H^K0kRknHF$r&GA)^O%* z>0-bzjgwEmAtB2BvNJmcFH`j@MZJywTheJMg$9-~OLa7IPDe+4T#ME5l?QU^uMso# zxdpuk`!aI&HW;Hq{RZPX|7E386dysFA5}_}Z?^4AVWQ&YUNrF}Qn?m{A|=ACtiJg8 zz&OTE{Qth+=Do}l+*!9zXxF)0mf}g;mbMXB(?3ntlx6bK?ZKHwt=HV6EW1hrgxB>i z!9>1y{D>t$5z0{2bT2=BKd+LR=&A#bjY(W~>C_Px%>D-Ox9gDqg|K_jtXb`0lzpW^ z@q41*&DX6L{=mtT(`)?f2g?m$=~iA)Wf*giLkkz2Kf5RA?v5ka-afQ_MiD;gDaF$% zT)RENSI|GSNf>QhXiqzBH}X05E-ZnnFd|oG1uxN6#HQ^Ov$&p z_dR!6z#?kZle`V5Wz|T(=w$Wjr+2Dsy%D+Kmf|*1`?|Zmk~WsMt^AKw(mved0z?Ks zMN1B%NXol0vx=iFe92#{qID=b57|rO{lh&(T;$P+l{%#uq3RMdY{`;|2u{5?{}4hl zN>bl=tJk>{JWJpi`UnT#+IKyq+;WZ!eimZ%mFPNxy=lF%7C_8hL29=Kj$F!m@(7_X zDMaXovXb$C7)DzaOMu+{!9iY^q$eSwRA8cl;_CW=gA(6dbY`n+Xq*c-|s{{k=#!-o=zEQ&A}V~YzrD3jt_&#jd*GwuD(jo zUuHya+1rw|k`$WNgu)S=l|G37cXoaQ-@)qEMvl~f;>wyu=Op$M3yX>Ln!hLpIj5py z>?0;qvLP8qd0(F06uM(U^`?RKYA}U{67k_zSeTQT>1FTuh9ebByI););KNl;Q~2`21#o^P4zmi^%#pKIyc=)t$(9 z)8hu+fYN0mtOOqv(uGoItk;`!yO$=C!E5Z9rbw^#`A#2heRFO!+_S&l*N|Cl=-rjG zkhg}KS_3(qNCGRyWq*9JDvImmMl2|q5pR?G?xflQYeSg8XsSgo7YZl1ra72zHWUpk; z*E_!civo2rhRM9H%8^F2r!}{_hSL9z4DN$@C^q~v>Y|ZzL07wA^9i`WUO=RJuDO z7`mw_x|f}PsM?zl#7d0)J$3;sG=r}S?>U`%lrNp^mAcU+ON-=LHfDsP?qPo<~JtsRep(6@gVh{^PiYsb zm50+e7$~j;rJ=$0xhRf*;|>jYbM=tJG)|G1U~FRvex(riSdsr{+JJ?0m0uea5zxpa z8y5dwuC4yL%X8$?jx8N$W{*!Rm!z1Y=pRoR%9D9RQiF6Ql8vwz^z_$H%&WirZij0Y z=gFk*tu2u--r>5Ts%FaZX?G;1gAcApt*8dVR95jti=~NL$WrGy^1Va*V5`;`0y}ul z&@l|0Szi)at;O7NJLfY#*21t-y%^e9K?8GI?jx&QnmhvSH26{68HPhB`rhKw!hNh( z=&ckpZEroHUKN*Y@!^E}w>#hI!O9b*jEVdjeA+)5;cuK#^Q^o(mj4#*E(V2FEN-a<+r7lTAx7 z*C_e_-X$f!1Ef7DMOnLpQAG>1)V12v()}up2695d4*Zy(8FqF!f-_1LZK|55FdVhA zMn;v2i~Q{5jGBik+H}lMdtN?cTC8$`23i?%AU}?kw9aJ*q&DFW5ekp?&fbDOfh(mJ z`^S#A;8g~f0)-%#Zx$36gX~c0za#L854f%#`{AFJz8@ImaAzJi)v36%_boibD%fP@ zeV>~GD5#seo#t&GmGw50KYMS{u(;1pN8y1NMVOaNIq@(&w7d`2xzYT*&*LhM?H^=b zt4`l}Tp1UX-z7*caaFf2S0zIAwm~%zO#v=JI33SvI3Q!tn0o5VkLs#c3Kf z&`ZrG=eh8{P=#KC6Z(qL0b|v&+0ma)JhPZ;7rxZYEnAmbT)|E&va-uSBp~m0v{_@QgLDV9M z@iya}n;XncCEDQI6D5Fzr6dlj-H2K_Hqs64g?&<$j^nm{ZUKea;kf?Qgpuk^Y#qO~ zop)U}E<1mr#h|Nn9oLDM5V=f)`%Eh-ntLej1R&KHuOX{8hYfJfLualleJqyJy-!U$ z-Ttit)W`=-=1;#m=hRbmT^Y)}b4pq-wQ*oTod*T`$o`VeO646^Xb#yW=%cJ<7p(dJ z+VmE4V^$MG+`(J_E>Yi(pL^5U^CByEs9Yqox*D*Z^tZOM8Y{CmT(xz~7B-g@5MG7bxg%`; z0YotSgwWH4r}uPCe(CO$(y`O&0IliP-P~9s=!GsY28o3OL}ytWNdCecb|Gwsd@m;T z6Q6m51s}Rq`r$Sw-!n@<$PVYdHTp=BQB8E+6n!m%6;gX~hCI$+XEe@)T+86&!*GE# z)%am%)8u-v=?)*iOaN(&KU{BVvZ9XU>Xsuy9JokAHYB8XnAy@tDC*Z;eKr^)C<}X3 z0=UMsm}?ni=2Yj>;LXYDH#9d5j0k6UW9-n64P zu4SNTs#xQ`>reBWW7C1ZI+ftS0t@O%)$Bs%ejd{@R^ZR-NLRHGQWO$_iVkWc;v?Hx zv(xq|755}6on+!585tDNwWoegd7i~>!u{X<``y>}K-u%c9LYfIlYCE@5}_bv$qTbK zHPfUY8zY=t0+4Qn+gNcF8Tk>bGnr_dE$bYbjS8tplZLQ)D>DHBbnU`I}Y zm+f$9XqKPDiV-S=_DHQh@pGO<^vCGd$>r(9r}4gnkNs(K4LXu26f8)la-rWoSTbDFFtYW526Cu3p5no@&QkTw$lbL>kWkk%f@HV zTMmSu2jXY>f9;mPdb1&@KBb>lE2`P9IPa24ipPoi6D0>wFeGUpG%H{ob(=GN?13KG z*4M_@o`eV&OcE7Ux;f;s#BN;GCH=a{n7l5ANVR&?Bo!Kz;2K@m7g}BQ1I?V0@*D1G zGa5}CjBomeUuY+H(p+Td_G~80zxloX6oU4YRs9;7|uZmCc?CZ?Tyt`ZOM0bxKmzy~^6Z!z(}= zDj#~XvtV8t=Ga3{ISL1&p>QBf5_+8mDs?_SFl~zwQp?nY3z!WsKw^lvx?P_=jkLpk zNUH6w>Xk-IW(8D4f>1$_-lPl2hfqMxJM0-~-O2SD+?T~&Z`@B&?x)`&xPZ%S3tJxUjyCVmrQ5P$F)p?=mwcFje zmQfrR!X3OD1#!NZ7AS439UME~hVVL8I{ob3&UG`*xw?bb>Qn{m9}xGG?75`Z}rLwP}`#2j(F@nmiFZs3cgmzK7#-E!T800wjV(!{?a>+R2iPl*Xsj z>Tj>C|5rCheS0uY!dr39?Fe_ICJ)Omf}dhlQT=uFQu}cPlF`5bqHrRNLUOSM?E>n{ zDwo(+0|OvRg9rtkDfo&)?=XS{Z(fH*LAzI(=CtqFt--a-2Lw~jA8k1^eM1iI#z=Yn z^dv$n2+C3nZTMi*I-s3Ox~|Fd|H6e;Ra7I|$%MWbqHF(_d($e?yB~`;nI1#Gm7oxY z;ZnOV+-ozRp(uBgkUXS8kYszUE&93w zxih_WF6)SdzVdoTxFAo^Q(mryVdn*7g;264!iR*Qp_=HZqnUFC+AQo;<=}WQcWtM#!YT@0~3ufUcj~z$T8gFR$hJFxMB4ZEXF!5DdFy z18#d4Iq<<$3W=8LBo~CfLlwDis;YGF0`Ml$K>haA@R6<`UriJAQbz*2IIFXa{^G)s zWMAtA3L|$2sT{7VU&n+Nx%xotaLWkA2;n~7I%JSMPVJPWT5KT_$}Ks7pjife-XKyJ znM(&2UB=aU_{01kciHl!_3;Yg3blT`x$W{iC2DnR&EzDPGiw@{foeC_7^nKIWB1cy zsZ+O1zWj>_s1xQ;OyQ;(_;p@tn%kShD8(n-$)erlnXU!DtBiN6V*t_@GR_Y2eYQH= zWXb~<7f1^r&M0oBk@@WqFq*Rnq>sJQ8~EuFMW|={`?+(nhFm99S%Id3o{nW(l`U~s zI6}Ay-lTZ}7MTq+0B#%BObD*N zrtVOrX2h)93E9~HYUyCYv@ZNe>=K(=hpyuRr6RId_r0$Nd~(b)nw$C*boE2_HMwZ{ zs=jA%CVK`5fwqjiX}2+EcX3f^SBiS83$9GY=8?W!I!Si35E?K#~E;VLQpj9>aD7 z=L}lxdeN&H%o5u)E8Z(=BJ-*6#p`I|3XTe9`b~arKe_LhsprLQ&4a9sE&7aYYXg6r z#>b7^blPjKp(H9>6F}*cMF(^W*$(w6;p9ofVi3?5ERWKYo-|2?jj86sW#axR=dZO~ zmZDewlw|(2Pj{#usF40%s!pO9VYmfeUSRYbWi+qoPHRi!shdEHPtSjtLyR~RU6k^$ zt+n4}$0&d$4z!kyZ!05s+TAvp8~%#}JyQZ7uZ^{}yq}qh;UT^@Bk+B5chhV?`qIsC zKYb1|ehKw5?6Fu0!Z`ckT=;-Ac>YSqJx!vRMW57JkKNDn+$FF zW56PvS+{RqFePc57qTl0Vv9=0W~Vl{f*Ozn5^xOXr0rB|v|q(t;jZub*&--pg#u68oLvG{RqUIR4~OJr z&{R-eJDH^0d2W5oCy$EW?1(py1>*xV_=gLn+)4_d?a0*PkF~&TrCg1*-3@g&Bx3hY zJ<<;R{-K%Qvuh13PUmgnic|;%CFtKdjdOK2>di7S#k8$Ilh)OhK(bD5g^w;VBnRO% zLLviP!|R@DN*^-wc&+$Er8~8T2KBr0403Z4P?U4zpv!bO1JN@$UED8R((%$3R=;Xk z&<2A`{Ajcs3JCeBe=w{?a8?mXSFJ$Jj-me)Y&XZgY@@exv=4UGlgS=TWd}z=V`P1@ z`M8b*bKGr1?;TW_;kiMnr=8KvG9deSXP{DnTHulUDX%yGQdsyvT9>Ce*|y0tborBS z7LVLZv5|Y+$tmca43&f;T6T<-nV)0G#GTbynsDWEH=P`8d{eQ-ZOcqgi-bLW3+`%D zDJPK6DE^eJ9eOD&F6h*g)Q-Nh?NWUT z`sYe$uLwNh5uwz;A)(<@z;1Hx-UaqlqL!`4cD07o3uP^%pJoK@x>mO`DlIVX;X5Xe zTNtQbCaI6FsTlV8cF-T&b{XQnMqp~s#93Q$Erc3fF!gwUx0 z_V9deQu`=-z5ynOjZ_^T%HL|>vS&qut|^Z(Gyx%`y^-83fIKI9IBN176+^OLYyRf2 zp6gyS&X2mB^~_kLYk*EJ+;y|#JoIQG-mnmBGmq4}A2JNo&*qixcvS&oGG+43)Z-Er zNbb=cEziBrI_38 z@J{ZXexe_y16wqIbWTXVUYoB3DA<9s#wyi^fo}6ZIoXhiCjG)!;v5_bOhIop=PnrM z)=Dc}ZgCTh`DIRodfhHYg$ikaL_!95_}3}mxO1iCw4~1tdW@D>eZTOl`XxqrTVIlG z(-XbBdJoV#!21i5c3onE0B`p}Uvn^+H(jg18%XX9q&zZA?<@wUxAkPa7He8W$n-BPyj(+0Fnu*y1G-xtwO2H=0c(@<&46b2rTKZ|$b<8s? zL=q!RFxL@@k{WMXb4uFcl{4cAr@8e_!zPW|I}#&_&}zsn=)=e&=SRlTXl~8Ll^g7O z6=ko>J!Eg&Ri*q6LlL*V`iRBJzfT_!8V;~I_ju?q!5UBicZP=;=iAH!jRDLW^0hq@ z=qgT*F=0R}Czv*4)fJv){|3xHEqd(ZuMq)sqdH@8kh=sX8hS%94!zhz1qmTKb?d9; z9$+*#m35Fjfj>BmOyqLvoQNf-XNr&VcvgwZTLGhM3txm zZwO>Bol}g@Zi)dUgeCe#zIvq}tAnyI=f;5nSE{!KjRg)A-l=x`IZ04q^xX<2+iM`7Z`5#lD0o8N2xLL;WcphGUspcQ|QO+ca2x?~1B%BFRx zxQ{Q|L_`P7aifwmb6GfiMT&jCL?!ShtIVatkD)39u!bu#-{~=q1w4Es8N*P)Myxu8 zaCKq&;KGbk%+ly#B9E=S%0ARVOb6zehnlyyLXLaW>ILAChlSbD^K@ye&`T=CN`?iD zxj0}73TwmTi4(lK-mPcF=6#}^dtMrq4b%ASs2Gq9ML6t%`9S(svHdq1>ysIun4B;= z6N`+RnBiU)j6~l zcjP=gSvGAM3r^pp#g~B>)C4SHXMXx%s=u=k5@ikCmVo*NBJJ-`+PWO5-pU{(=lA7E=XFe$c@l40 zZ`QCQJrp7)uJ5ExX%zMnQHW4(c$W6^4`XTn%Dq!56cD_e{1O`0 zyCs#F)5^zm6U~o{sNy5`r)YWVqiAhCThi$PU_9E+y+c7g|QN2cGl+oJY#^kJ^K3H$F78AL@G18X z3RGKz@Vn)1pX%>%A$B>Pet2Wm^pqrXl=u0JW70e63~1g|UjGzCJfV9R-o$k1M3Q3D+=r+wkabgdz3D%*wEf>0^CNZ!3EN~Ug=HPo5sSBh4|zM$ z8Jq=A*wyc9|G0Qd)J*|4dfwp$U^~KVS4B+L@R%7b#byS2VYsHR#AasEt$tOsAe6Og z{TyeV5g?oF;TX9F_0xCK$F*T4oap=WL`;wXGJOdE6Ds+|4r=j|v^`NTQ!DS8o~$Si z3#KbJJe9P~<+8`r3BuGL#3T*B4hqzAw804L=NSxO+z&<2PJ;ZdgjU^hD!+!g_%?SZ zc-YsED$LXtEE5Bk_;`tM$*erMp@nCfYvn=qlAdUpW*+c#&2^*5^PQ8h3_ol7e!&U$ z;R9mYieGobe59^4Gpzv%XOekgdXj^!vqxIZE4)mvh*>&YVsWj9h{vk5 z)1!`v-X%n4>}i61-eA4b-~u%y2tBMP9n8|`B@94NfBI*myxd^gs6*R zFmXlqHsTauW0Huy}86bx4CFt JcJ}Ju{|{f(_nQC! literal 0 HcmV?d00001 diff --git a/docs/src/developer/introduction.md b/docs/src/developer/introduction.md new file mode 100644 index 000000000..4796167e6 --- /dev/null +++ b/docs/src/developer/introduction.md @@ -0,0 +1 @@ +# Developer Documentation diff --git a/docs/src/user/basic_setup.md b/docs/src/user/basic_setup.md new file mode 100644 index 000000000..dc1c1d466 --- /dev/null +++ b/docs/src/user/basic_setup.md @@ -0,0 +1,99 @@ +# Basic Setup + +This section will explain how to setup youki for use. Currently the only way to get youki is to compile it from the source. Currently youki only supports Linux systems, so this documentation assumes you are using a Linux system. For running youki on other platforms, you can use Vagrant, which is explained in the last part of this section. + +### Requirements + +youki is written in rust, so you will need rust toolchain installed to compile it. Also if you want to use it with a higher level container engine, such as docker, you will need to install that as well. In case you want to use one of the sub-crates of youki as a dependency, you may not need docker. + +The rest of document uses docker as an example when required. + +- Rust(See [here](https://www.rust-lang.org/tools/install)), edition 2021 +- Docker(See [here](https://docs.docker.com/engine/install)) + +Apart from these basic requirements, some other libraries are also required to compile and run youki. To install them on : + +#### Debian, Ubuntu and related distributions + +```console +$ sudo apt-get install \ + pkg-config \ + libsystemd-dev \ + libdbus-glib-1-dev \ + build-essential \ + libelf-dev \ + libseccomp-dev +``` + +#### Fedora, Centos, RHEL and related distributions + +```console +$ sudo dnf install \ + pkg-config \ + systemd-devel \ + dbus-devel \ + elfutils-libelf-devel \ + libseccomp-devel +``` + +#### Getting the source + +After installing the dependencies you will need to clone the youki repository if you want to use it directly : + +```console +git clone git@github.com:containers/youki.git +``` + +Or if you want ot use it as a dependency in a Rust project, you can specify it in your Cargo.toml : + +```toml +[dependencies] +... +liboci-cli = { git = "https://github.com/containers/youki.git" } +... +``` + +You can specify the crate that you need as a dependency in place of `liboci-cli` + +#### Installing the source + +If you have cloned the source, you can build it using + +```console +# go into the cloned directory +cd youki + +# build +./build.sh +``` + +This will build the youki, and put the binary at the root level of the cloned directory. + +When using it as a dependency, you can use it in your source as : + +``` +use liboci_cli::{...} +``` + +You can specify the crate that you need as a dependency in place of `liboci-cli` + +#### Using Vagrant to run Youki on non-Linux Platform + +you can use the vagrantfile provided with the source, to setup vagrant and run youki inside it. You can see [vagrant installation](https://www.vagrantup.com/docs/installation) for how to download and setup vagrant. + +Once done, you can run vagrant commands in the cloned directory to run youki inside the VM created by vagrant : + +```console +# for rootless mode, which is default +vagrant up +vagrant ssh + +# or if you want to develop in rootful mode +VAGRANT_VAGRANTFILE=Vagrantfile.root vagrant up +VAGRANT_VAGRANTFILE=Vagrantfile.root vagrant ssh + +# in virtual machine +cd youki +./build.sh + +``` diff --git a/docs/src/user/basic_usage.md b/docs/src/user/basic_usage.md new file mode 100644 index 000000000..560c2cb0c --- /dev/null +++ b/docs/src/user/basic_usage.md @@ -0,0 +1,110 @@ +# Basic Usage + +This section will explain how to use youki as a low-level container runtime with some other high-level container runtime such as docker. + +Youki can also be used with other runtimes such as podman, but for this section, we will use docker as an example. + +#### Using youki with a high-level runtime + +This explains how to use youki as a lowe level runtime along with some high-level runtime. For this example we use Docker. + +1. If you have the docker daemon running (which you probably have if you have installed docker), first you will need to stop it. For example, if you use systemd as your init system, you can use `systemctl stop docker`, with root permissions. +2. Start the docker daemon with you as the runtime : + + ```console + dockerd --experimental --add-runtime="youki=$(pwd)/youki" # run in the root directory + ``` + + This will start the docker daemon with youki as the low-level runtime. This will keep the terminal busy, so you can start it as a background process or open a new terminal for next steps. + + In case you get an error message such as : + + ``` + failed to start daemon: pid file found, ensure docker is not running or delete /var/run/docker.pid + ``` + + This means you docker daemon is already running, and you need to stop is as explained in step 1. + +3. Now you can use docker normally, and give youki as the runtime in the arguments : + ```console + docker run -it --rm --runtime youki busybox # run a container + ``` +4. After you are done, you can stop the docker daemon process that was started in step 2, and restart the normal docker daemon, by using your init system, for example using `systemctl start docker` with root permissions, for systems using systemd. + +#### Using Youki Standalone + +You can use youki directly without a higher level runtime, but it will be tedious. This example shows how to create and run a container using only youki. + +1. run + + ```console + mkdir -p tutorial/rootfs + ``` + + This will create a directory which will be used as the root directory for the container. + +2. run + ```console + cd tutorial + docker export $(docker create busybox) | tar -C rootfs -xvf - + ``` + This will export the basic file system structure needed to run the container. You can manually create all the directories, but it will be tedious. +3. run + ```console + ../youki spec + ``` + This will generate the config.json file needed to setup the permissions and configuration for the container process. + You can manually edit the file to customize the behavior of the container process. For example, you can edit the process.args to specify command to run in the process : + ```json + "process": { + ... + "args": [ + "sleep", "30" + ], + ... + } + ``` + Then go back to the root directory : + ```console + cd .. + ``` +4. After this you can use youki to : + - create the container + ```console + # create a container with name `tutorial_container` + sudo ./youki create -b tutorial tutorial_container + ``` + - get the state of the container + ```console + # you can see the state the container is `created` + sudo ./youki state tutorial_container + ``` + - start the container + ```console + # start the container + sudo ./youki start tutorial_container + ``` + - list all the containers + ```console + # will show the list of containers, the container is `running` + sudo ./youki list + ``` + - delete the specific container + ```console + # delete the container + sudo ./youki delete tutorial_container + ``` +5. The above step created the containers with root permissions, but youki can also create rootless containers,which does not need root permissions. for that, after exporting the rootfs from docker, when creating spec, use `--rootless` flag : + ```console + ../youki spec --rootless + ``` + This will generate the spec needed for rootless containers. + After this the steps are same, except you can run them without sudo and root access : + ```console + cd .. + sudo ./youki create -b tutorial rootless_container + sudo ./youki state rootless_container + sudo ./youki start rootless_container + sudo ./youki list + sudo ./youki delete rootless_container + ``` diff --git a/docs/src/user/crates.md b/docs/src/user/crates.md new file mode 100644 index 000000000..3a6320b2b --- /dev/null +++ b/docs/src/user/crates.md @@ -0,0 +1,3 @@ +# Crates provided + +The github repo of youki is a Cargo workspace containing several crates, which provide various functionalities : for example, libcgroups provide means to work with Linux cgroups. Youki itself is one of the crates, and uses functionalities from the other crates. The following sections explain the crates and public interfaces provided by them, as well as commandline interface of youki itself. diff --git a/docs/src/user/introduction.md b/docs/src/user/introduction.md new file mode 100644 index 000000000..6dd167c39 --- /dev/null +++ b/docs/src/user/introduction.md @@ -0,0 +1,13 @@ +# User Documentation + +This section is the user documentation for youki, for using it as a low-level container runtime. +This section will explain : + +- Basic Setup : Explains how to setup youki for use +- Basic Usage : Using it as a low level runtime +- crates +- libcgroups +- libcontainer +- liboci-cli +- libseccomp +- youki diff --git a/docs/src/user/libcgroups.md b/docs/src/user/libcgroups.md new file mode 100644 index 000000000..c1320c333 --- /dev/null +++ b/docs/src/user/libcgroups.md @@ -0,0 +1,66 @@ +# libcgroups + +`libcgroups` provide a Rust interface over cgroups v1 and v2. + +It exposes the modules : + +- common +- stats +- systemd +- test_manager +- v1 +- v2 + +### common + +This module contains common features for cgroups v1 and v2, such as + +- trait CgroupManager, which gives public interface to + + - add a task to a cgroup + - apply resource restriction + - remove a cgroup + - freezer cgroup state control + - get stats from a cgroup + - get pids belonging to the cgroup + +- functions `write_cgroup_file_str` and `write_cgroup_file` which writes data to a cgroup file +- function `read_cgroup_file` which reads data from given cgroup file +- function `get_cgroup_setup` which returns setup of cgroups (v1,v2, hybrid) on the system + +### stats + +This module contains structs and functions to work with statistics for a cgroup, such as + +- struct Stats which contains all following individual stat structs + - CpuStats : stores usage and throttling information + - MemoryStats : stores usage of memory, swap and memory combined, kernel memory, kernel tcp memory and other memory stats + - PidStats : contains current number of active pids and allowed number of pids + - BlkioStats : contains block io related stats such as number of bytes transferred from/to by a device in cgroup, number of io operations done by a device in cgroup, device access and queue information + - HugeTlbStats : stats for huge TLB , containing usage, max_usage and fail count +- function `supported_page_size` which returns hugepage size supported by the system +- functions to operate with data in cgroups files such as + - `parse_single_value` : reads file expecting it to have a single value, and returns the value + - `parse_flat_keyed_data` : parses cgroup file data which is in flat keyed format (key value) + - `parse_nested_keyed_data` : parses cgroup file data which is in nested keyed format (key list of values) + - `parse_device_number` : parses major and minor number of device + +### systemd + +This module contains functions and modules to deal with systemd, as currently youki depends on systemd. This exposes + +- function `booted` to check if the system was booted with systemd or not +- module controller_type, which contains + - enum ControllerType which is used to specify controller types +- module manager, which contains + - struct Manager, which is the cgroup manager, and contain information about the root cgroups path, path for the specific cgroups, client to communicate with systemd etc. This also implements CgroupManager trait. + +### test_manager + +This exposes a TestManager struct which can be used as dummy for testing purposes, which also implements CgroupManager. + +### v1 and v2 + +These modules contains modules and fucntions related to working with cgroups v1 and v2. They expose respective cgroups version specific mangers, and some utility functions to get mount points (for v1 and v2), get subsystem mount points (for v1) and get available controllers (for v2) etc. + +For cgroups v2, it also exposes devices module, which gives functions for working with bpf such as load a bpf program, query info of a bpf program, attach and detach a bpf program to a cgroup, etc. diff --git a/docs/src/user/libcontainer.md b/docs/src/user/libcontainer.md new file mode 100644 index 000000000..5d1455c7a --- /dev/null +++ b/docs/src/user/libcontainer.md @@ -0,0 +1,35 @@ +# libcontainer + +This is a library that provides utilities to setup and run containers. Youki itself uses this crate to manage and control the containers. + +This exposes several modules, each dealing with a specific aspect of working with containers. + +- apparmor : functions that deal with apparmor, which is a Linux Kernel security module to control program capabilities with per program profiles. + +- capabilities : this has functions related to set and reset specific capabilities, as well as to drop extra privileges. + +- config : this exposes YoukiConfig struct, which contains a subset of the data in the config.json. This subset is needed when starting or managing containers after creation, and rather than parsing and passing around whole config.json, this smaller YoukiConfig is passed, which is comparatively faster. + +- container : This is the core of the container module, and contains modules and structs that deal with the container lifecycle including creating, starting , stopping and deleting containers. + +- hooks : exposes function run_hooks, which is used to run various container lifecycle hooks as specified in oci-spec + +- namespaces : exposes Namespaces struct, which deals with applying namespaces to a container process + +- notify_socket : this has NotifyListener struct, which is used internally to communicate between the main youki process and the forked container process + +- process : a module which exposes functions related to forking the process, starting the container process with correct namespaces and setting up the namespaces + +- rootfs : this contains modules which deal with rootfs, which is minimal filesystem + +- rootless : this deals with running containers rootless, that is without needing root privileges on the host system + +- seccomp : this deals with setting up seccomp for container process, this uses libseccomp. + +- signal : this provide simple wrappers for unix signal's ascii names/numbers. + +- syscall : this provides a trail Syscall, which is used to abstract over several functions which need to call libc functions, so that other parts of library can use them without having to deal with implementation details. + +- tty : this deals with setting up the tty for the container process + +- utils : provides various utility functions such as parse_env to parse the env variables, do_exec to do an exec syscall and execute a binary, get_cgroups_path, create_dir_all_with_mode etc. diff --git a/docs/src/user/liboci_cli.md b/docs/src/user/liboci_cli.md new file mode 100644 index 000000000..7fa5bac34 --- /dev/null +++ b/docs/src/user/liboci_cli.md @@ -0,0 +1,24 @@ +# liboci-cli + +This is a crate to parse command line arguments for OCI container runtimes as specified in the OCI Runtime Command Line Interface. This exposes structures with Clap Parser derived, so that they can be directly used for parsing oci-commandline arguments. + +### Implemented subcommands + +| Command | liboci-cli | CLI Specification | runc | crun | youki | +| :--------: | :--------: | :---------------: | :--: | :--: | :---: | +| create | ✅ | ✅ | ✅ | ✅ | ✅ | +| start | ✅ | ✅ | ✅ | ✅ | ✅ | +| state | ✅ | ✅ | ✅ | ✅ | ✅ | +| kill | ✅ | ✅ | ✅ | ✅ | ✅ | +| delete | ✅ | ✅ | ✅ | ✅ | ✅ | +| checkpoint | | | ✅ | ✅ | | +| events | ✅ | | ✅ | | ✅ | +| exec | ✅ | | ✅ | ✅ | ✅ | +| list | ✅ | | ✅ | ✅ | ✅ | +| pause | ✅ | | ✅ | ✅ | ✅ | +| ps | ✅ | | ✅ | ✅ | ✅ | +| restore | | | ✅ | ✅ | | +| resume | ✅ | | ✅ | ✅ | ✅ | +| run | ✅ | | ✅ | ✅ | ✅ | +| spec | ✅ | | ✅ | ✅ | ✅ | +| update | | | ✅ | ✅ | | diff --git a/docs/src/user/libseccomp.md b/docs/src/user/libseccomp.md new file mode 100644 index 000000000..521aeb24c --- /dev/null +++ b/docs/src/user/libseccomp.md @@ -0,0 +1,3 @@ +# libseccomp + +This crate provides Rust FFI bindings to [libseccomp](https://github.com/seccomp/libseccomp). This is adapted from code generated using rust-bindgen from a C header file. This also manually fixes some of the issues that occur as rust-bindgen has some issues when dealing with C function macros. diff --git a/docs/src/youki.md b/docs/src/youki.md new file mode 100644 index 000000000..6fd7749b8 --- /dev/null +++ b/docs/src/youki.md @@ -0,0 +1,39 @@ +# Youki + +

+ +

+ +youki is an implementation of the [OCI runtime-spec](https://github.com/opencontainers/runtime-spec) in Rust, similar to [runc](https://github.com/opencontainers/runc). + +## About the name + +youki is pronounced as /joʊki/ or yoh-key. +youki is named after the Japanese word 'youki', which means 'a container'. In Japanese language, youki also means 'cheerful', 'merry', or 'hilarious'. + +## Motivation + +Here is why we are writing a new container runtime in Rust. + +- Rust is one of the best languages to implement the oci-runtime spec. Many very nice container tools are currently written in Go. However, the container runtime requires the use of system calls, which requires a bit of special handling when implemented in Go. This is too tricky (e.g. _namespaces(7)_, _fork(2)_); with Rust, it's not that tricky. And, unlike in C, Rust provides the benefit of memory safety. While Rust is not yet a major player in the container field, it has the potential to contribute a lot: something this project attempts to exemplify. +- youki has the potential to be faster and use less memory than runc, and therefore work in environments with tight memory usage requirements. Here is a simple benchmark of a container from creation to deletion. + + | Runtime | Time (mean ± σ) | Range (min … max) | + | :-----: | :----------------: | :-----------------: | + | youki | 198.4 ms ± 52.1 ms | 97.2 ms … 296.1 ms | + | runc | 352.3 ms ± 53.3 ms | 248.3 ms … 772.2 ms | + | crun | 153.5 ms ± 21.6 ms | 80.9 ms … 196.6 ms | + +
+ Details about the benchmark + + - A command used for the benchmark + ```console + $ hyperfine --prepare 'sudo sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' --warmup 10 --min-runs 100 'sudo ./youki create -b tutorial a && sudo ./youki start a && sudo ./youki delete -f a' + ``` + - Enviroment + `console $ ./youki info Version 0.0.1 Kernel-Release 5.11.0-41-generic Kernel-Version #45-Ubuntu SMP Fri Nov 5 11:37:01 UTC 2021 Architecture x86_64 Operating System Ubuntu 21.04 Cores 12 Total Memory 32025 Cgroup setup hybrid Cgroup mounts blkio /sys/fs/cgroup/blkio cpu /sys/fs/cgroup/cpu,cpuacct cpuacct /sys/fs/cgroup/cpu,cpuacct cpuset /sys/fs/cgroup/cpuset devices /sys/fs/cgroup/devices freezer /sys/fs/cgroup/freezer hugetlb /sys/fs/cgroup/hugetlb memory /sys/fs/cgroup/memory net_cls /sys/fs/cgroup/net_cls,net_prio net_prio /sys/fs/cgroup/net_cls,net_prio perf_event /sys/fs/cgroup/perf_event pids /sys/fs/cgroup/pids unified /sys/fs/cgroup/unified CGroup v2 controllers cpu detached cpuset detached hugetlb detached io detached memory detached pids detached device attached Namespaces enabled mount enabled uts enabled ipc enabled user enabled pid enabled network enabled cgroup enabled $ ./youki --version youki version 0.0.1 commit: 0.0.1-0-0be33bf $ runc -v runc version 1.0.0-rc93 commit: 12644e614e25b05da6fd08a38ffa0cfe1903fdec spec: 1.0.2-dev go: go1.13.15 libseccomp: 2.5.1 $ crun --version crun version 0.19.1.45-4cc7 commit: 4cc7fa1124cce75dc26e12186d9cbeabded2b710 spec: 1.0.0 +SYSTEMD +SELINUX +APPARMOR +CAP +SECCOMP +EBPF +CRIU +YAJL ` +
+ +- The development of [railcar](https://github.com/oracle/railcar) has been suspended. This project was very nice but is no longer being developed. This project is inspired by it. +- I have fun implementing this. In fact, this may be the most important. From 784ca705ad22ad256b0e37588a82cd17ba8a102c Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Wed, 22 Dec 2021 21:39:41 +0530 Subject: [PATCH 2/5] improve the user documentation --- docs/src/user/basic_setup.md | 68 ++++++----- docs/src/user/basic_usage.md | 220 +++++++++++++++++++--------------- docs/src/user/crates.md | 6 +- docs/src/user/introduction.md | 20 ++-- docs/src/user/libcgroups.md | 55 +++++---- docs/src/user/libcontainer.md | 30 ++--- docs/src/user/liboci_cli.md | 2 +- 7 files changed, 232 insertions(+), 169 deletions(-) diff --git a/docs/src/user/basic_setup.md b/docs/src/user/basic_setup.md index dc1c1d466..2bc748b17 100644 --- a/docs/src/user/basic_setup.md +++ b/docs/src/user/basic_setup.md @@ -1,17 +1,18 @@ # Basic Setup -This section will explain how to setup youki for use. Currently the only way to get youki is to compile it from the source. Currently youki only supports Linux systems, so this documentation assumes you are using a Linux system. For running youki on other platforms, you can use Vagrant, which is explained in the last part of this section. +This explains the requirements for compiling Youki as a binary, to use it as a low-level container runtime, or to depend once of its crates as dependency for your own project. -### Requirements +Youki currently only supports Linux Platfrom, and to use it on other platform you will need to use some kind of virtualization. The repo itself provides Vagrantfile that provides basic setup to use Youki on non-Linux system using Vagrant. The last sub-section explains using this vagrantfile. + +Also note that Youki currently only supports and expects systemd as init system, and would not work on other systems. There is currently work on-going to put systemd dependent features behind a feature flag, but till then you will need a systemd enabled system to work with Youki. -youki is written in rust, so you will need rust toolchain installed to compile it. Also if you want to use it with a higher level container engine, such as docker, you will need to install that as well. In case you want to use one of the sub-crates of youki as a dependency, you may not need docker. +### Requirements -The rest of document uses docker as an example when required. +As Youki is written in Rust, you will need to install and setup Rust toolchain to compile it. The instructions for that can be found on Rust's official site [here](https://www.rust-lang.org/tools/install). -- Rust(See [here](https://www.rust-lang.org/tools/install)), edition 2021 -- Docker(See [here](https://docs.docker.com/engine/install)) +You can use Youki by itself to start and run containers, but it can be a little tedious, as it is a low-level container runtime. You can use a High-level container runtime, with its runtime set to Youki, so that it will be easier to use. Both of these are explained in the [Basic Usage](./basic_usage.md). For using it along with an high-level runtime, you will to install one such as Docker or Podman. This documentation uses Docker in its examples, which can be installed from [here](https://docs.docker.com/engine/install). -Apart from these basic requirements, some other libraries are also required to compile and run youki. To install them on : +To compile and run, Youki itself depends on some underlying libraries being installed. You can install them using your respective package manager as shown below. #### Debian, Ubuntu and related distributions @@ -36,28 +37,23 @@ $ sudo dnf install \ libseccomp-devel ``` -#### Getting the source +--- -After installing the dependencies you will need to clone the youki repository if you want to use it directly : +### Getting the source -```console -git clone git@github.com:containers/youki.git -``` +Currently Youki can only be installed from the source code itself, so you will need to clone the Youki GitHub repository to get the source code for using it as a runtime. If you are using any crates of Youki as dependency you need to do this step, as Cargo will automatically clone the repository for you. -Or if you want ot use it as a dependency in a Rust project, you can specify it in your Cargo.toml : +To clone the repository, run -```toml -[dependencies] -... -liboci-cli = { git = "https://github.com/containers/youki.git" } -... +```console +git clone git@github.com:containers/youki.git ``` -You can specify the crate that you need as a dependency in place of `liboci-cli` +This will create a directory named youki in the directory you ran the command in. This youki directory will be referred to as root directory throughout the documentation. -#### Installing the source +### Installing the source -If you have cloned the source, you can build it using +Once you have cloned the source, you can build it using ```console # go into the cloned directory @@ -67,23 +63,40 @@ cd youki ./build.sh ``` -This will build the youki, and put the binary at the root level of the cloned directory. +This will build the Youki binary, and put it at the root level of the cloned directory, that is in the youki/ . -When using it as a dependency, you can use it in your source as : +--- + +### Using sub-crates as dependency + +To use any of the sub-crate as a dependency in your own project, you can specify the dependency as follows, + +```toml +[dependencies] +... +liboci-cli = { git = "https://github.com/containers/Youki.git" } +... +``` + +Here we use `liboci-cli` as an example, which can be replaced by the sub-crate that you need. + +Then you can use it in your source as ``` use liboci_cli::{...} ``` -You can specify the crate that you need as a dependency in place of `liboci-cli` +--- -#### Using Vagrant to run Youki on non-Linux Platform +### Using Vagrant to run Youki on non-Linux Platform -you can use the vagrantfile provided with the source, to setup vagrant and run youki inside it. You can see [vagrant installation](https://www.vagrantup.com/docs/installation) for how to download and setup vagrant. +As explained before, Youki only support Linux, and to build/use it on non-Linux Platforms, you will need to use some kind of virtualization. The repo provides a Vagrantfile to do the required VM setup using Vagrant, which can be installed from [here](https://www.vagrantup.com/docs/installation). -Once done, you can run vagrant commands in the cloned directory to run youki inside the VM created by vagrant : +Once installed ans setup, you can run vagrant commands in the cloned directory to run Youki inside the VM created by vagrant : ```console +# in the youki directory + # for rootless mode, which is default vagrant up vagrant ssh @@ -95,5 +108,4 @@ VAGRANT_VAGRANTFILE=Vagrantfile.root vagrant ssh # in virtual machine cd youki ./build.sh - ``` diff --git a/docs/src/user/basic_usage.md b/docs/src/user/basic_usage.md index 560c2cb0c..477d9c591 100644 --- a/docs/src/user/basic_usage.md +++ b/docs/src/user/basic_usage.md @@ -1,110 +1,142 @@ # Basic Usage -This section will explain how to use youki as a low-level container runtime with some other high-level container runtime such as docker. +This explains using Youki as a low-level container runtime. Youki can be used by itself to create, start and run containers, but doing so can be tedious, and thus you might want to use a higher-level runtime with Youki set as its runtime, so that you can get a convenient and easy interface. -Youki can also be used with other runtimes such as podman, but for this section, we will use docker as an example. +You can use Youki with Docker, or Podman, but for the purpose of the examples, we will illustrate using Docker. + +Youki can run in two modes, namely rootful mode, and rootless mode. The primary difference from the user-perspective in these is that as the name suggests, rootless mode does not require root/admin permissions, while rootful mode needs the root permissions. Both of these are shown in the examples below. #### Using youki with a high-level runtime -This explains how to use youki as a lowe level runtime along with some high-level runtime. For this example we use Docker. +We will first see how to use Youki with a high-level runtime such as Docker. You can install Docker from [here](https://www.vagrantup.com/docs/installation). + +By default, after installation the docker sets up so that its daemon process will start running in background after booting up. By default, this configures Docker to use its default low-level runtime, and to use Youki instead , we will first need to stop the running Docker daemon. + +As Youki needs systemd to compile, this assumes that you are running on a systemd based system. So you an first check if the docker daemon is running or not by running + +```console +systemctl status docker +``` + +This will print a message showing if the daemon is active or not. If it is active, then you will need to stop it by running + +```console +sudo systemctl stop docker +``` + +After this you need to manually restart the docker daemon, but with Youki as its runtime. To do this, run following command in the youki/ directory after building youki + +```console +dockerd --experimental --add-runtime="youki=$(pwd)/youki" # run in the youki directory +``` + +This will start the daemon and hang up the console. You can either start this as a background process to continue using the same terminal, or use another terminal, which will make it easier to stop the docker daemon later. -1. If you have the docker daemon running (which you probably have if you have installed docker), first you will need to stop it. For example, if you use systemd as your init system, you can use `systemctl stop docker`, with root permissions. -2. Start the docker daemon with you as the runtime : +In case you don't stop the original daemon, you can get an error message after previous command - ```console - dockerd --experimental --add-runtime="youki=$(pwd)/youki" # run in the root directory - ``` +``` +failed to start daemon: pid file found, ensure docker is not running or delete /var/run/docker.pid +``` - This will start the docker daemon with youki as the low-level runtime. This will keep the terminal busy, so you can start it as a background process or open a new terminal for next steps. +Now that the docker daemon is running, you can use docker normally as you will, but you will be able to specify Youki as its low-level runtime to actually create, start and stop the containers. - In case you get an error message such as : +You can try running a container such as - ``` - failed to start daemon: pid file found, ensure docker is not running or delete /var/run/docker.pid - ``` +```console +docker run -it --rm --runtime youki busybox # run a container +``` - This means you docker daemon is already running, and you need to stop is as explained in step 1. +This will start a busybox container, and give access to terminal inside it. -3. Now you can use docker normally, and give youki as the runtime in the arguments : - ```console - docker run -it --rm --runtime youki busybox # run a container - ``` -4. After you are done, you can stop the docker daemon process that was started in step 2, and restart the normal docker daemon, by using your init system, for example using `systemctl start docker` with root permissions, for systems using systemd. +After you are done, you can stop the docker daemon by sending it a signal, either by using `Ctrl` + `C` if you are running the process in another terminal, or by using kill command with the pid of it, if you have started it as a background process. + +Then to start the original/normal Docker daemon, you can run + +```console +sudo systemctl start docker +``` #### Using Youki Standalone -You can use youki directly without a higher level runtime, but it will be tedious. This example shows how to create and run a container using only youki. - -1. run - - ```console - mkdir -p tutorial/rootfs - ``` - - This will create a directory which will be used as the root directory for the container. - -2. run - ```console - cd tutorial - docker export $(docker create busybox) | tar -C rootfs -xvf - - ``` - This will export the basic file system structure needed to run the container. You can manually create all the directories, but it will be tedious. -3. run - ```console - ../youki spec - ``` - This will generate the config.json file needed to setup the permissions and configuration for the container process. - You can manually edit the file to customize the behavior of the container process. For example, you can edit the process.args to specify command to run in the process : - ```json - "process": { - ... - "args": [ - "sleep", "30" - ], - ... - } - ``` - Then go back to the root directory : - ```console - cd .. - ``` -4. After this you can use youki to : - - create the container - ```console - # create a container with name `tutorial_container` - sudo ./youki create -b tutorial tutorial_container - ``` - - get the state of the container - ```console - # you can see the state the container is `created` - sudo ./youki state tutorial_container - ``` - - start the container - ```console - # start the container - sudo ./youki start tutorial_container - ``` - - list all the containers - ```console - # will show the list of containers, the container is `running` - sudo ./youki list - ``` - - delete the specific container - ```console - # delete the container - sudo ./youki delete tutorial_container - ``` -5. The above step created the containers with root permissions, but youki can also create rootless containers,which does not need root permissions. for that, after exporting the rootfs from docker, when creating spec, use `--rootless` flag : - ```console - ../youki spec --rootless - ``` - This will generate the spec needed for rootless containers. - After this the steps are same, except you can run them without sudo and root access : - ```console - cd .. - sudo ./youki create -b tutorial rootless_container - sudo ./youki state rootless_container - sudo ./youki start rootless_container - sudo ./youki list - sudo ./youki delete rootless_container - ``` +Youki can also be used directly, without a higher-level runtime such as Docker to create, start, stop and delete the container, but the process can be tedious. Here we will show how you can do that, to run a simple container with desired program running in it. + +Note that we will still be using Docker to generate the rootfs required for running the container. + +To start, in the youki directory, make another directory named tutorial, and craete a sub-directory rootfs inside it + +```console +mkdir -p tutorial/rootfs +``` + +After that, you will need to use docker to create the required directory structure + +```console +cd tutorial +docker export $(docker create busybox) | tar -C rootfs -xvf - +``` + +This will create the required directory structure for using it as a root directory inside the container. + +Now the any container runtime gets the information about the permissions, configurations and constraints for the container process by using a config.json file. Youki has a command which can generate the default config for you. To do this, run + +```console +../youki spec +``` + +After this, you can manually edit the file to customize the behavior of the container process. For example, to run the desired program inside the container, you can edit the process.args + +```json +"process": { +... +"args": [ + "sleep", "30" +], +... + } +``` + +Here you can change the args to specify the program to be run, and arguments to be given to it. + +After this, go back to the youki/ directory + +```console +cd .. +``` + +As the setup is complete, you can now use youki to create the container, start the container, get its state etc. + +```console +# create a container with name `tutorial_container` +sudo ./youki create -b tutorial tutorial_container + +# you can see the state the container is `created` +sudo ./youki state tutorial_container + +# start the container +sudo ./youki start tutorial_container + +# will show the list of containers, the container is `running` +sudo ./youki list + +# delete the container +sudo ./youki delete tutorial_container +``` + +The example above shows how to run Youki in a 'rootful' way. To run it without root permissions, that is, in rootless mode, few chagnes are required. + +First, after exporting the rootfs from docker, while generating the config, you will need to pass the rootless flag. This will generate the config withe the options needed for rootless operation of the container. + +```console +../youki spec --rootless +``` + +After this, the steps are basically the same, except you do not need to use sudo while running youki. + +```console +cd .. +./youki create -b tutorial rootless_container +./youki state rootless_container +./youki start rootless_container +./youki list +./youki delete rootless_container +``` diff --git a/docs/src/user/crates.md b/docs/src/user/crates.md index 3a6320b2b..b2179e7fa 100644 --- a/docs/src/user/crates.md +++ b/docs/src/user/crates.md @@ -1,3 +1,7 @@ # Crates provided -The github repo of youki is a Cargo workspace containing several crates, which provide various functionalities : for example, libcgroups provide means to work with Linux cgroups. Youki itself is one of the crates, and uses functionalities from the other crates. The following sections explain the crates and public interfaces provided by them, as well as commandline interface of youki itself. +Youki repo itself is a Cargo workspace, comprising of several sub-crates, each one for some specific functionality. The youki binary depends on this to provide the low-level functionality, and you can use these crate as a dependency for your own projects as well. + +For more information on how to add a sub-crate as a dependency in your project, see [Basic Usage](./basic_usage.md). + +The sub-section in this part briefly explains some of the crates, and some of the functionality they expose. This should be good enough to get a general idea of each crate. To get a detailed information about the structs, functions and modules each crate exposes, unfortunately, for the time being, you will need to go through the source itself ; but we are working on creating better docs. diff --git a/docs/src/user/introduction.md b/docs/src/user/introduction.md index 6dd167c39..7b089757e 100644 --- a/docs/src/user/introduction.md +++ b/docs/src/user/introduction.md @@ -1,13 +1,13 @@ # User Documentation -This section is the user documentation for youki, for using it as a low-level container runtime. -This section will explain : +THis section provides documentation of youki and the sub crates that the youki repo contains for the users. So if you are using youki as a low level container runtime, or using any of the crates in youki workspace as dependencies for your own project, this section might be helpful for you. -- Basic Setup : Explains how to setup youki for use -- Basic Usage : Using it as a low level runtime -- crates -- libcgroups -- libcontainer -- liboci-cli -- libseccomp -- youki +This is divided into following sub-sections : + +- Basic Setup : This explains how the dependencies and setup required to compile and run youki +- Basic Usage : This explains using youki itself as a low-level container runtime, or using one of the crates as dependencies +- crates : This section provides brief explanation of the member crates of youki repo workspace + - libcgroups + - libcontainer + - liboci-cli + - libseccomp diff --git a/docs/src/user/libcgroups.md b/docs/src/user/libcgroups.md index c1320c333..c061476ed 100644 --- a/docs/src/user/libcgroups.md +++ b/docs/src/user/libcgroups.md @@ -1,8 +1,8 @@ # libcgroups -`libcgroups` provide a Rust interface over cgroups v1 and v2. +libcgroups is the crate that contains functionality to work with Linux cgroups. This provide an easy to use interface over reading and writing cgroups files, as well as various structs that represent the cgroups data. -It exposes the modules : +The modules that it exposes are : - common - stats @@ -11,11 +11,13 @@ It exposes the modules : - v1 - v2 +Following is a short explanation of these modules + ### common -This module contains common features for cgroups v1 and v2, such as +This module contains functionality that is general to any type of cgroup. SOme of the things this provides are -- trait CgroupManager, which gives public interface to +- trait CgroupManager : this gives and interface for the following - add a task to a cgroup - apply resource restriction @@ -24,43 +26,56 @@ This module contains common features for cgroups v1 and v2, such as - get stats from a cgroup - get pids belonging to the cgroup -- functions `write_cgroup_file_str` and `write_cgroup_file` which writes data to a cgroup file +- functions `write_cgroup_file_str` and `write_cgroup_file` which write data to a cgroup file - function `read_cgroup_file` which reads data from given cgroup file - function `get_cgroup_setup` which returns setup of cgroups (v1,v2, hybrid) on the system ### stats -This module contains structs and functions to work with statistics for a cgroup, such as +This module has functionalities related to statistics data of the cgroups, and struts representing it. + +Some of the things it exposes are + +- struct `Stats` which contains following individual structs + + - CpuStats : contains cpu usage and throttling information + + - MemoryStats : contains usage of memory, swap and memory combined, kernel memory, kernel tcp memory and other memory stats -- struct Stats which contains all following individual stat structs - - CpuStats : stores usage and throttling information - - MemoryStats : stores usage of memory, swap and memory combined, kernel memory, kernel tcp memory and other memory stats - PidStats : contains current number of active pids and allowed number of pids - - BlkioStats : contains block io related stats such as number of bytes transferred from/to by a device in cgroup, number of io operations done by a device in cgroup, device access and queue information - - HugeTlbStats : stats for huge TLB , containing usage, max_usage and fail count + + - BlkioStats : contains block io related stats, such as : number of bytes transferred from/to by a device in cgroup, number of io operations done by a device in cgroup, device access and queue information etc. + + - HugeTlbStats : containing stats for Huge TLB such as usage, max_usage and fail count + - function `supported_page_size` which returns hugepage size supported by the system -- functions to operate with data in cgroups files such as + +- utility functions to operate with data in cgroups files such as + - `parse_single_value` : reads file expecting it to have a single value, and returns the value + - `parse_flat_keyed_data` : parses cgroup file data which is in flat keyed format (key value) + - `parse_nested_keyed_data` : parses cgroup file data which is in nested keyed format (key list of values) + - `parse_device_number` : parses major and minor number of device ### systemd -This module contains functions and modules to deal with systemd, as currently youki depends on systemd. This exposes +THis is the module used by youki to interact with sytemd, and it exposes several functions to interact - function `booted` to check if the system was booted with systemd or not -- module controller_type, which contains - - enum ControllerType which is used to specify controller types -- module manager, which contains - - struct Manager, which is the cgroup manager, and contain information about the root cgroups path, path for the specific cgroups, client to communicate with systemd etc. This also implements CgroupManager trait. + +- module controller_type, which contains `enum ControllerType` which is used to specify cgroup controllers available on a system + +- module manager, which contains `struct Manager`, which is the cgroup manager, and contain information such as the root cgroups path, path for the specific cgroups, client to communicate with systemd etc. This also implements CgroupManager trait, and thus can be used for cgroups related operations. ### test_manager -This exposes a TestManager struct which can be used as dummy for testing purposes, which also implements CgroupManager. +This exposes a TestManager struct which can be used as dummy for cgroup testing purposes, which also implements CgroupManager. ### v1 and v2 -These modules contains modules and fucntions related to working with cgroups v1 and v2. They expose respective cgroups version specific mangers, and some utility functions to get mount points (for v1 and v2), get subsystem mount points (for v1) and get available controllers (for v2) etc. +These two modules contains functionalities specific to cgroups version 1 and version 2. Both of these expose respective cgroup managers, which can be used to manage that type of cgroup, as well as sme utility functions related to respective cgroup version, such as `get_mount_points` (for v1 and v2), `get_subsystem_mount points (for v1) and `get_available_controllers` (for v2) etc. -For cgroups v2, it also exposes devices module, which gives functions for working with bpf such as load a bpf program, query info of a bpf program, attach and detach a bpf program to a cgroup, etc. +The v2 module also exposes devices module, which gives functionality for working with bpf ; such as load a bpf program, query info of a bpf program, attach and detach a bpf program to a cgroup, etc. diff --git a/docs/src/user/libcontainer.md b/docs/src/user/libcontainer.md index 5d1455c7a..be293779f 100644 --- a/docs/src/user/libcontainer.md +++ b/docs/src/user/libcontainer.md @@ -1,35 +1,35 @@ # libcontainer -This is a library that provides utilities to setup and run containers. Youki itself uses this crate to manage and control the containers. +THis crate provides functionality for creating and managing containers. Youki itself uses this crate to manage and control the containers. This exposes several modules, each dealing with a specific aspect of working with containers. - apparmor : functions that deal with apparmor, which is a Linux Kernel security module to control program capabilities with per program profiles. -- capabilities : this has functions related to set and reset specific capabilities, as well as to drop extra privileges. +- capabilities : this has functions related to setting and resetting specific capabilities, as well as to drop extra privileges from container process. -- config : this exposes YoukiConfig struct, which contains a subset of the data in the config.json. This subset is needed when starting or managing containers after creation, and rather than parsing and passing around whole config.json, this smaller YoukiConfig is passed, which is comparatively faster. +- config : this exposes YoukiConfig struct, which contains a subset of the data in the config.json. This is the subset that is needed when starting or managing containers after creation, and rather than parsing and passing around whole config.json, this smaller YoukiConfig is passed, which is comparatively faster. -- container : This is the core of the container module, and contains modules and structs that deal with the container lifecycle including creating, starting , stopping and deleting containers. +- container : This is the core of the container module, and contains sub-modules and structs that deal with the container lifecycle including creating, starting, stopping and deleting containers. -- hooks : exposes function run_hooks, which is used to run various container lifecycle hooks as specified in oci-spec +- hooks : exposes function run_hooks, which is used to run various container lifecycle hooks as specified in oci-spec. -- namespaces : exposes Namespaces struct, which deals with applying namespaces to a container process +- namespaces : exposes Namespaces struct, which deals with applying namespaces to a container process. -- notify_socket : this has NotifyListener struct, which is used internally to communicate between the main youki process and the forked container process +- notify_socket : this has NotifyListener struct, which is used internally to communicate between the main youki process and the forked container processes. -- process : a module which exposes functions related to forking the process, starting the container process with correct namespaces and setting up the namespaces +- process : a module which exposes functions related to forking the process, setting up the namespaces and starting the container process with correct namespaces. -- rootfs : this contains modules which deal with rootfs, which is minimal filesystem +- rootfs : this contains modules which deal with rootfs, which is minimal filesystem that is provided to the container. -- rootless : this deals with running containers rootless, that is without needing root privileges on the host system +- rootless : this deals with running containers in a rootless configuration, that is running containers without needing root permissions. -- seccomp : this deals with setting up seccomp for container process, this uses libseccomp. +- seccomp : this deals with setting up seccomp for container process, and this this uses libseccomp crate in turn to do that. -- signal : this provide simple wrappers for unix signal's ascii names/numbers. +- signal : this provide simple wrappers for unix signal, so that parsing them from their names or signal numbers is easier. -- syscall : this provides a trail Syscall, which is used to abstract over several functions which need to call libc functions, so that other parts of library can use them without having to deal with implementation details. +- syscall : this provides a trait Syscall, which is used to abstract over several functionalities which need to call libc functions. This allows the other parts of library to use those functions without having to deal with implementation details. -- tty : this deals with setting up the tty for the container process +- tty : this deals with setting up the tty for the container process. -- utils : provides various utility functions such as parse_env to parse the env variables, do_exec to do an exec syscall and execute a binary, get_cgroups_path, create_dir_all_with_mode etc. +- utils : provides various utility functions such as `parse_env` to parse the env variables, `do_exec` to do an exec syscall and execute a binary in the container process, `get_cgroups_path`, `create_dir_all_with_mode` etc. diff --git a/docs/src/user/liboci_cli.md b/docs/src/user/liboci_cli.md index 7fa5bac34..939e70ebd 100644 --- a/docs/src/user/liboci_cli.md +++ b/docs/src/user/liboci_cli.md @@ -1,6 +1,6 @@ # liboci-cli -This is a crate to parse command line arguments for OCI container runtimes as specified in the OCI Runtime Command Line Interface. This exposes structures with Clap Parser derived, so that they can be directly used for parsing oci-commandline arguments. +This module provides the structs for command line arguments for OCI container runtimes as specified in the OCI Runtime Command Line Interface The exposed structures are with Clap Parser derived, so that they can be directly used for parsing oci-commandline arguments. ### Implemented subcommands From e987d7811efbc3df2d14bce9bc4fbe7fa8f77ebc Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Thu, 23 Dec 2021 20:01:40 +0530 Subject: [PATCH 3/5] Initial dev documentation WIP --- docs/doc-draft.md | 3 +- docs/src/SUMMARY.md | 12 + docs/src/assets/control_flow.drawio.svg | 696 ++++++++++++++++++ docs/src/developer/basics.md | 17 + .../developer/crate_specific_information.md | 5 + docs/src/developer/documentation_mdbook.md | 18 + docs/src/developer/good_places_to_start.md | 23 + docs/src/developer/integration_test.md | 5 + docs/src/developer/introduction.md | 14 + docs/src/developer/libcgroups.md | 9 + docs/src/developer/libcontainer.md | 36 + docs/src/developer/liboci_cli.md | 5 + docs/src/developer/libseccomp.md | 5 + docs/src/developer/test_framework.md | 5 + docs/src/developer/unwritten_rules.md | 23 + docs/src/developer/youki.md | 9 + 16 files changed, 883 insertions(+), 2 deletions(-) create mode 100644 docs/src/assets/control_flow.drawio.svg create mode 100644 docs/src/developer/basics.md create mode 100644 docs/src/developer/crate_specific_information.md create mode 100644 docs/src/developer/documentation_mdbook.md create mode 100644 docs/src/developer/good_places_to_start.md create mode 100644 docs/src/developer/integration_test.md create mode 100644 docs/src/developer/libcgroups.md create mode 100644 docs/src/developer/libcontainer.md create mode 100644 docs/src/developer/liboci_cli.md create mode 100644 docs/src/developer/libseccomp.md create mode 100644 docs/src/developer/test_framework.md create mode 100644 docs/src/developer/unwritten_rules.md create mode 100644 docs/src/developer/youki.md diff --git a/docs/doc-draft.md b/docs/doc-draft.md index e77558c2a..762cae9d6 100644 --- a/docs/doc-draft.md +++ b/docs/doc-draft.md @@ -1,4 +1,4 @@ -_This is a draft for a high level documentation of Youki. After it is finished this is intended to explain how control flow and high level functioning of Youki happens for development purposes. +\_This is a draft for a high level documentation of Youki. After it is finished this is intended to explain how control flow and high level functioning of Youki happens for development purposes. ## Some reference links @@ -78,7 +78,6 @@ The main youki process will set up pipes used as message passing and synchroniza Note: clone(2) offers us the ability to enter into user and pid namespace by creatng only one process. However, clone(2) can only create new pid namespace, but cannot enter into existing pid namespaces. Therefore, to enter into existing pid namespaces, we would need to fork twice. Currently, there is no getting around this limitation. - - [fork(2) man page](https://man7.org/linux/man-pages/man2/fork.2.html) - [clone(2) man page](https://man7.org/linux/man-pages/man2/clone.2.html) - [pid namespace man page](https://man7.org/linux/man-pages/man7/pid_namespaces.7.html) diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 1642b6c5e..10de2053d 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -16,3 +16,15 @@ --- - [Developer Documentation](./developer/introduction.md) + - [Basics](./developer/basics.md) + - [Unwritten Rule](./developer/unwritten_rules.md) + - [Good places to start](./developer/good_places_to_start.md) + - [This Documentation](./developer/documentation_mdbook.md) + - [Crate Specific Information](./developer/crate_specific_information.md) + - [libcgroups](./developer/libcgroups.md) + - [libcontainer](./developer/libcontainer.md) + - [liboci-cli](./developer/liboci_cli.md) + - [libseccomp](./developer/libseccomp.md) + - [youki](./developer/youki.md) + - [test_framework](./developer/test_framework.md) + - [integration_test](./developer/integration_test.md) diff --git a/docs/src/assets/control_flow.drawio.svg b/docs/src/assets/control_flow.drawio.svg new file mode 100644 index 000000000..505da318f --- /dev/null +++ b/docs/src/assets/control_flow.drawio.svg @@ -0,0 +1,696 @@ + + + + + + + +
+
+
+ Intermediate Process +
+
+
+ + + Intermediate Pr... + + + + + + + + + +
+
+
+ youki create +
+
+
+ + + youki create + + + + + + + +
+
+
+ + Init +
+ Process + +
+
+
+ + + Init... + + + + + + + + + +
+
+
+ youki start +
+
+
+ + + youki start + + + + + + + + + +
+
+
+ High-level runtime +
+
+
+ + + High-level... + + + + + + + +
+
+
+ + youki create $id + +
+
+
+ + + youki create $id + + + + + + + + + +
+
+
+ + + fork(2) + + +
+
+
+ + + fork(2) + + + + + + + + + +
+
+
+ + send identifier mapping request + +
+
+
+ + + send identifier mapping request + + + + + + + +
+
+
+ unshare(CLONE_NEWUSER) +
+
+
+ + + unshare(CLONE_NEWUSER) + + + + + + + + + +
+
+
+ write uid mapping +
+
+
+ + + write uid mapping + + + + + + + +
+
+
+ write gid mapping +
+
+
+ + + write gid mapping + + + + + + + + + +
+
+
+ + send mapping written + +
+
+
+ + + send mapping written + + + + + + + + + +
+
+
+ + + fork(2) + + +
+
+
+ + + fork(2) + + + + + + + + + +
+
+
+ + send ready +
+ +
+
+
+ + + send ready + + + + + + + +
+
+
+ + send ready with a init process pid + +
+
+
+ + + send ready with a init process... + + + + + + + + + +
+
+
+ update the pid file +
+
+
+ + + update the pid file + + + + + + + +
+
+
+ exit +
+
+
+ + + exit + + + + + + + + + +
+
+
+ High-level runtime +
+
+
+ + + High-level... + + + + + + + +
+
+
+ + youki start $id + +
+
+
+ + + youki start $id + + + + + + + + + +
+
+
+ + send the start signal + +
+
+
+ + + send the start signal + + + + + + + +
+
+
+ exit +
+
+
+ + + exit + + + + + + + + + + +
+
+
+ setup cgroup +
+
+
+ + + setup cgroup + + + + + + + + +
+
+
+ unshare(CLONE_NEWPID) +
+
+
+ + + unshare(CLONE_NEWPID) + + + + + + + +
+
+
+ set uid and gid +
+
+
+ + + set uid and gid + + + + + + + + + +
+
+
+ exit +
+
+
+ + + exit + + + + + + + +
+
+
+ exec the container entry point +
+
+
+ + + exec the container entry point + + + + + + + +
+
+
+ wait for the start signal +
+
+
+ + + wait for the start signal + + + + + + + +
+
+
+ setup capability +
+
+
+ + + setup capability + + + + + + + +
+
+
+ pivot_root(2) +
+
+
+ + + pivot_root(2) + + + + + + + +
+
+
+ unshare(rest of NAMESPACE) +
+
+
+ + + unshare(rest of NAMESPACE) + + + + + + + +
+
+
+ exit +
+
+
+ + + exit + + + + + + + +
+
+
+ setup seccomp +
+
+
+ + + setup seccomp + + + + + + + + + +
+
+
+ + send seccomp notify fd + +
+
+
+ + + send seccomp notify fd + + + + + + + + + +
+
+
+ + send seccomp notify done +
+ +
+
+
+ + + send seccomp notify done + + + + + + + + + +
+
+
+ + send seccomp notify through listener +
+ +
+
+
+ + + send seccomp notify through listene... + + + + + + + +
+
+
+ seccomp agent +
+
+
+ + + seccomp age... + + + + + + + + + Viewer does not support full SVG 1.1 + + + + \ No newline at end of file diff --git a/docs/src/developer/basics.md b/docs/src/developer/basics.md new file mode 100644 index 000000000..84441ce3e --- /dev/null +++ b/docs/src/developer/basics.md @@ -0,0 +1,17 @@ +# Basics + +This section has the basic information and resources needed to work with any part of youki. This also assumes that you already know Rust. If you don't yet, you should probably learn it first before contributing here. Good resources for that can be found on the Rust's [official site](https://www.rust-lang.org/learn). + +Youki is a low level container runtime, which aims to deal with creation and management of Linux containers on a low level. Some of other such low-level runtimes are [runc](https://github.com/opencontainers/runc) and [crun](https://github.com/containers/crun). These are usually used by a higher-level runtime to actually create and manage containers, while the higher level runtime provides a much easier interface for users. + +Before you start working on developing youki, you should go through the User documentation as it specifies the requirements and setup for running youki. For developing youki, you will need to install the dependencies and clone the repo, as specified in the [Basic Setup](../user/basic_setup.md) and [Basic Usage](../user/basic_usage.md) sections. + +## Resources + +The youki is an OCI-spec compliant runtime. OCI or Open Container Initiative provides a common specification to be used by runtimes, so that there would be an uniform interface, and users or other programs need not deal with each runtime's interface separately. Their main GitHub page is at [https://github.com/opencontainers](https://github.com/opencontainers), and information about the specifications can be found there. + +The specification of the runtime itself can be found at [https://github.com/opencontainers/runtime-spec/blob/master/runtime.md](https://github.com/opencontainers/runtime-spec/blob/master/runtime.md). + +Another good resource is man pages, one main place where they can be found is at [https://man7.org/](https://man7.org/). These have very good explanation of the programming interfaces for Linux libraries and various kernel features. To find information about a certain function or feature, you can search `man ` in a search engine, or you can search at the [site itself](https://man7.org/linux/man-pages/index.html). These come in handy when dealing with low level system interfaces. + +Happy developing!!! diff --git a/docs/src/developer/crate_specific_information.md b/docs/src/developer/crate_specific_information.md new file mode 100644 index 000000000..4fa094967 --- /dev/null +++ b/docs/src/developer/crate_specific_information.md @@ -0,0 +1,5 @@ +# Crate Specific Information + +This part contains resources and information for each individual sub-crate which is part of the Cargo workspace of youki. + +In case you are working with some specific crate, you can find resources about it in its page. Also make sure you add any resources that you find when working on them as well. diff --git a/docs/src/developer/documentation_mdbook.md b/docs/src/developer/documentation_mdbook.md new file mode 100644 index 000000000..576b58bb1 --- /dev/null +++ b/docs/src/developer/documentation_mdbook.md @@ -0,0 +1,18 @@ +# This Documentation + +This documentation is generated using mdbook. you can check out the [mdbook documentation](https://rust-lang.github.io/mdBook/) for more info on mdbook. + +TODO, update after deciding the final way to host the mdbook +To compile and deploy this mdbook follow the instructions + +``` +git worktree add /tmp/book -b gh-pages +mdbook build +rm -rf /tmp/book/* # this won't delete the .git directory +cp -rp book/* /tmp/book/ +cd /tmp/book +git add -A +git commit 'new book message' +git push origin gh-pages +cd - +``` diff --git a/docs/src/developer/good_places_to_start.md b/docs/src/developer/good_places_to_start.md new file mode 100644 index 000000000..5caecfc92 --- /dev/null +++ b/docs/src/developer/good_places_to_start.md @@ -0,0 +1,23 @@ +# Good places to start + +First of all, welcome to youki! Hope you have fun while developing and contributing :) + +This lists some of the known places that are long-running and would be useful for beginners. But as the things under development can change any time, the best place to check for is the issues on [GitHub repo](https://github.com/containers/youki/issues). You can find issues with labels `good fist issue` or `help wanted` and start working on them. + +Other that that you can search for `TODO` or `FIXME` comments in the source, and try working on them, but some of them can be hard issues. + +--- + +This lists known sections that can be good for beginners at the time of the writing. Please update as things change. + +#### Documentation Comments + +Currently youki is decently commented, and those explain most of the public facing API and structs. But there are still places which can use more doc comments, and examples showing usage, so people can use the docs generated by `cargo doc` as a guide. + +If you don't know much about container runtimes or low level system working, then this can be a good place to start, as while going through the code an documenting it, you can learn about it. Make sure that you update this documentation with useful links that you found while commenting some code if it has some peculiar behavior. + +#### Integration Tests + +You can find more detailed information about this in the integration_test crate, but in brief, we currently use [OCI-runtime-tools](https://github.com/opencontainers/runtime-tools) provided integration tests to validate that youki performs as per the OCI spec. But those are written in Go, which makes the developer depend on two language env to compile youki and test it ; as well as those have some issue which make it hard to run them on some local systems. + +Thus we are porting those test to Rust, so that it can be an Rust implementation of OCI-runtime integration tests, and easy to run on locacl systems. If you know Go and Rust this can be a great place to start. Check out the [tracking issue](https://github.com/containers/youki/issues/361). diff --git a/docs/src/developer/integration_test.md b/docs/src/developer/integration_test.md new file mode 100644 index 000000000..fe83183e4 --- /dev/null +++ b/docs/src/developer/integration_test.md @@ -0,0 +1,5 @@ +# integration_test + +This crate is Rust port of OCI-runtime tools validation integration tests. This is still a work in progress, and you can check the [tracking issue](https://github.com/containers/youki/issues/361) for the current progress. + +TODO add explanation of reasons behind this, and a brief explanation of the crate implementation diff --git a/docs/src/developer/introduction.md b/docs/src/developer/introduction.md index 4796167e6..66dc81b55 100644 --- a/docs/src/developer/introduction.md +++ b/docs/src/developer/introduction.md @@ -1 +1,15 @@ # Developer Documentation + +This is the documentation of various parts of youki for those who wish to contribute in the development of youki. + +First of all, Thank you! If you have any issues or doubts, you can ask them on youki's discord server [here](https://discord.gg/h7R3HgWUct). + +This is split into pars as below : + +- Basics : contains resources and information that you will probably need for modifying any part of the code + +- Unwritten Rules : This is the section to write down rules or conventions that are decided in PR discussions + +- Good Places to Start : This will give you some easy-entry points to start contributing + +- Crate specific Information : This sections contains sub-sections for each crate in youki workspace, and will have resources specific to that crate diff --git a/docs/src/developer/libcgroups.md b/docs/src/developer/libcgroups.md new file mode 100644 index 000000000..5271a8466 --- /dev/null +++ b/docs/src/developer/libcgroups.md @@ -0,0 +1,9 @@ +# libcgroups + +This crate provides and interface for working with cgroups in Linux. cgroups or control groups are Linux kernel feature which can be used to fine-control resources and permissions given to a particular process or a group of processes. You can read more about them on the [cgroups man page](https://man7.org/linux/man-pages/man7/cgroups.7.html). + +The initial version of cgroups is called the version 1 was implemented in kernel 2.6.24 , later in kernel version 4.5, a new version of cgroups was released, aimed to solve issues with v1, and was the version v2. + +This crate exposes modules to work with both of the version, as well as structs needed to store cgroups data. Apart from these two modules, cgroups crate also exposes utility functions to read cgroups files and parse their data, and a stats module that exposes the structs and functions to parse and store the statistics related to cgroups, such as cpu and memory usage, Huge TLB size and hits etc. + +As youki currently depends on systemd as an init system, this crate also exposes module systemd, which provides interface for working with systemd related operations. [systemd resource control](https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html) is a good place to read more about systemd and its involvement in resource control. diff --git a/docs/src/developer/libcontainer.md b/docs/src/developer/libcontainer.md new file mode 100644 index 000000000..ae2c5456f --- /dev/null +++ b/docs/src/developer/libcontainer.md @@ -0,0 +1,36 @@ +# libcontainer + +This is one of the core crates part of the youki workspace. This deals with the actual creation and management of the container processes, and provides functions and structs for the same. + +Remember, that in the end, a container is just another process in Linux, which has control groups, namespaces, pivot_root and other mechanisms applied to it. The program executing has the impression that is is running on a complete system, but from the host system's perspective, it is just another process, and has attributes such as pid, file descriptors, etc. associated with it like any other process. + +When given the create command, Youki will load the specification, configuration, sockets etc., use clone syscall to create the container process (init process), applies the limits, namespaces, and etc. to the cloned container process. The container process will wait on a unix domain socket before executing the command/program. + +The main youki process will setup pipes to communicate and synchronize with the intermediate and init process. The init process will notify the intermediate process, and then intermediate process to the main youki process that it is ready and start to wait on a unix domain socket. The youki process will then write the container state and exit. + +The following are some resources that can help understand with various Linux features used in the code of this crate. + +- [mio Token definition](https://docs.rs/mio/0.7.11/mio/struct.Token.html) +- [oom-score-adj](https://dev.to/rrampage/surviving-the-linux-oom-killer-2ki9) +- [unshare man page](https://man7.org/linux/man-pages/man1/unshare.1.html) +- [user-namespace man page](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) +- [wait man page](https://man7.org/linux/man-pages/man3/wait.3p.html) + +The main youki process creates the intermediate process and the intermediate process creates the container process (init process). The hierarchy is: `main youki process -> intermediate process -> init process` + +The main youki process will set up pipes used as message passing and synchronization mechanism with the init process. The reason youki needs to create/fork two process instead of one is due to the user and pid namespaces. In rootless container, we need to first enter user namespace, since all other namespaces requires CAP_SYSADMIN. When unshare or set_ns into pid namespace, only the children of the current process will enter into a different pid namespace. As a result, we must first fork a process to enter into user namespace, call unshare or set_ns for pid namespace, then fork again to enter into the correct pid namespace. + +Note: clone(2) offers us the ability to enter into user and pid namespace by creatng only one process. However, clone(2) can only create new pid namespace, but cannot enter into existing pid namespaces. Therefore, to enter into existing pid namespaces, we would need to fork twice. Currently, there is no getting around this limitation. + +- [fork(2) man page](https://man7.org/linux/man-pages/man2/fork.2.html) +- [clone(2) man page](https://man7.org/linux/man-pages/man2/clone.2.html) +- [pid namespace man page](https://man7.org/linux/man-pages/man7/pid_namespaces.7.html) + +This has functions related to set and reset specific capabilities, as well as to drop extra privileges + +- [Simple explanation of capabilities](https://blog.container-solutions.com/linux-capabilities-in-practice) +- [man page for capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) + +This has functions related to setting of namespaces to the calling process + +- [CLONE_NEWUSER flag](https://man7.org/linux/man-pages/man2/clone.2.html) diff --git a/docs/src/developer/liboci_cli.md b/docs/src/developer/liboci_cli.md new file mode 100644 index 000000000..14eca6ebe --- /dev/null +++ b/docs/src/developer/liboci_cli.md @@ -0,0 +1,5 @@ +# liboci-cli + +This crate was separated from original youki crate, and now contains a standalone implementation of structs needed for parsing commandline arguments for OCI-spec compliant runtime commandline interface. + +This primarily uses the crate clap-v3 for parsing the actual commandline arguments given to the runtime. diff --git a/docs/src/developer/libseccomp.md b/docs/src/developer/libseccomp.md new file mode 100644 index 000000000..6807e2a92 --- /dev/null +++ b/docs/src/developer/libseccomp.md @@ -0,0 +1,5 @@ +# libseccomp + +Seccomp is a linux kernel feature that allows a process a one-way transition into secure mode, where restrictions are applied to the syscalls the process can make, as well as restrictions on the file descriptors. + +This crate does not actually implement any particular feature, but provides Rust FFI bindings for seccomp module. These are primarily generated by using rsut-bindgen on seccomp C header file, and then manually fixed where any issues were found. diff --git a/docs/src/developer/test_framework.md b/docs/src/developer/test_framework.md new file mode 100644 index 000000000..a20dc4ca8 --- /dev/null +++ b/docs/src/developer/test_framework.md @@ -0,0 +1,5 @@ +# test_framework + +This is the framework specifically developed to implement the ported integration tests. This exposes Structs to represent individual tests, test groups and test managers which runs the test groups. This also exposes a trait which can be used to implement a custom test struct or a custom test group. + +By default the test groups are run in parallel using [crossbeam crate](https://www.crates.io/crates/crossbeam), and the default test_group implementation also runs individual tests parallelly. Sometimes you might need to run the individual test in certain order, serially such as when testing container lifecycle. In such cases you will need to implement the TestableGroup trait in a custom struct so you can finely control the order of execution. diff --git a/docs/src/developer/unwritten_rules.md b/docs/src/developer/unwritten_rules.md new file mode 100644 index 000000000..33b0ae307 --- /dev/null +++ b/docs/src/developer/unwritten_rules.md @@ -0,0 +1,23 @@ +# Unwritten Rule + +This is the place to write down rules or conventions that are discussed in PRs, so that newcomers can easily find them, without having to go through the PR history. So if you decide on any convention to follow for the project, please make sure to add them here. + +## Conventions to follow + +#### Errors + +Youki currently uses [anyhow](https://www.crates.io/crates/anyhow) library to deal with errors occurring while its execution. So wherever you use fallible actions, or functions that can return `Result`, make sure you attach enough information with the errors so that error logs can be useful for debugging later. For example, if you are reading a file, or parsing something and the operation does not succeed, you can add the path from which you attempted to read the file, or the string that you attempted to parse. + +Also for the error messages, we follow the convention all small-case letters and no period at the end, as discussed in [this PR](https://github.com/containers/youki/issues/313). Whenever you write error messages, please follow this convention to keep them uniform. + +#### Logs + +youki uses [log](https://crates.io/crates/log) crate to log information while running. Whenever adding code to interact with system or kernel faetures or such, make sure to add debug logs so that if youki crashes, you can trace the errors using logs. + +#### Comments + +Make sure that you comment copiously, and explain the peculiar behavior of your code so that others can understand why certain code is written the way it is. Also make sure to add doc comments and examples for `pub` items in the crates, so that users can find it from the docs generated by `cargo doc`. + +#### Update This Documentation + +Keep this Documentation updated ! Make sure you add any relevant doc-links and resources to this as you add code, so that it can help newcomers as well as others to find the resources in one single place. diff --git a/docs/src/developer/youki.md b/docs/src/developer/youki.md new file mode 100644 index 000000000..ad3ea3f1c --- /dev/null +++ b/docs/src/developer/youki.md @@ -0,0 +1,9 @@ +# youki + +This is the core crate that contains the youki binary itself. The simple control flow of youki can be explained as : + +

+ +

+ +TODO Add explanation here From d47580ec1352c0124ae15e336bbfbefe7043f6fc Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Fri, 24 Dec 2021 18:47:37 +0530 Subject: [PATCH 4/5] Update the dev documentation --- docs/src/developer/basics.md | 16 +++--- .../developer/crate_specific_information.md | 4 +- docs/src/developer/documentation_mdbook.md | 4 +- docs/src/developer/good_places_to_start.md | 12 ++--- docs/src/developer/integration_test.md | 8 ++- docs/src/developer/introduction.md | 16 +++--- docs/src/developer/libcgroups.md | 27 ++++++++-- docs/src/developer/libcontainer.md | 51 ++++++++++++------- docs/src/developer/liboci_cli.md | 4 +- docs/src/developer/libseccomp.md | 4 +- docs/src/developer/test_framework.md | 6 ++- docs/src/developer/unwritten_rules.md | 8 +-- docs/src/developer/youki.md | 18 ++++++- 13 files changed, 122 insertions(+), 56 deletions(-) diff --git a/docs/src/developer/basics.md b/docs/src/developer/basics.md index 84441ce3e..49c6a6eb0 100644 --- a/docs/src/developer/basics.md +++ b/docs/src/developer/basics.md @@ -1,17 +1,21 @@ # Basics -This section has the basic information and resources needed to work with any part of youki. This also assumes that you already know Rust. If you don't yet, you should probably learn it first before contributing here. Good resources for that can be found on the Rust's [official site](https://www.rust-lang.org/learn). +This section has the general information and resources needed to work with any part of youki. As youki is written in Rust, you should know some basic Rust before. If you don't yet, some good resources for that can be found on the Rust's [official site](https://www.rust-lang.org/learn). -Youki is a low level container runtime, which aims to deal with creation and management of Linux containers on a low level. Some of other such low-level runtimes are [runc](https://github.com/opencontainers/runc) and [crun](https://github.com/containers/crun). These are usually used by a higher-level runtime to actually create and manage containers, while the higher level runtime provides a much easier interface for users. +## Youki -Before you start working on developing youki, you should go through the User documentation as it specifies the requirements and setup for running youki. For developing youki, you will need to install the dependencies and clone the repo, as specified in the [Basic Setup](../user/basic_setup.md) and [Basic Usage](../user/basic_usage.md) sections. +Youki is a low level container runtime, which deals with the creation and management of Linux containers. Some of other such low-level runtimes are [runc](https://github.com/opencontainers/runc) and [crun](https://github.com/containers/crun). These are usually used by a higher-level runtime such as Docker or Podman to actually create and manage containers, where the higher level runtime provides a much easier interface for users. + +Before you start working on developing youki, you should go through [the User documentation](../user/introduction) as it specifies the requirements and setup for running youki. For developing youki, you will need to install the dependencies and clone the repo, as specified in the [Basic Setup](../user/basic_setup.md) and [Basic Usage](../user/basic_usage.md) sections. ## Resources -The youki is an OCI-spec compliant runtime. OCI or Open Container Initiative provides a common specification to be used by runtimes, so that there would be an uniform interface, and users or other programs need not deal with each runtime's interface separately. Their main GitHub page is at [https://github.com/opencontainers](https://github.com/opencontainers), and information about the specifications can be found there. +#### OCI + +Open containers initiative is project, which provides a standardization and standardized specification for operating-system-level virtualization. That way components that confirm to the specification provided by OCI spec, can interoperate with each other easily, and developing of new applications becomes easier. For example youuki can be used inplace of runc in Docker, as all three : Docker, runc and youki are OCI compliant, and have a standard interface. -The specification of the runtime itself can be found at [https://github.com/opencontainers/runtime-spec/blob/master/runtime.md](https://github.com/opencontainers/runtime-spec/blob/master/runtime.md). +Their main GitHub page is at [https://github.com/opencontainers](https://github.com/opencontainers), and more information about the runtime specifications can be found at [https://github.com/opencontainers/runtime-spec/blob/master/runtime.md](https://github.com/opencontainers/runtime-spec/blob/master/runtime.md). -Another good resource is man pages, one main place where they can be found is at [https://man7.org/](https://man7.org/). These have very good explanation of the programming interfaces for Linux libraries and various kernel features. To find information about a certain function or feature, you can search `man ` in a search engine, or you can search at the [site itself](https://man7.org/linux/man-pages/index.html). These come in handy when dealing with low level system interfaces. +As youki needs to deal with a lot of low level programming interfaces of Linux Kernel, another good place know is the online man pages project, which can be found at [https://man7.org/](https://man7.org/). Man pages provide detailed information about the programming interfaces of various features of Linux Kernel. You can simply search `man ` using a search engine, or you can search at the site itself, at [https://man7.org/linux/man-pages/index.html](https://man7.org/linux/man-pages/index.html). These can be very helpful to know about the behavior and usage of and reasoning behind various kernel features used throughout youki. Happy developing!!! diff --git a/docs/src/developer/crate_specific_information.md b/docs/src/developer/crate_specific_information.md index 4fa094967..7f5ddbf21 100644 --- a/docs/src/developer/crate_specific_information.md +++ b/docs/src/developer/crate_specific_information.md @@ -1,5 +1,5 @@ # Crate Specific Information -This part contains resources and information for each individual sub-crate which is part of the Cargo workspace of youki. +This section contains sub-sections for each individual crate in the youki workspace. Each of the seb-section will have information and resources on that particular crate. -In case you are working with some specific crate, you can find resources about it in its page. Also make sure you add any resources that you find when working on them as well. +In case you are working with some specific crate, you can find resources about it in its section. Also make sure you add any resources that you find when working on them as well. diff --git a/docs/src/developer/documentation_mdbook.md b/docs/src/developer/documentation_mdbook.md index 576b58bb1..a978ce6b8 100644 --- a/docs/src/developer/documentation_mdbook.md +++ b/docs/src/developer/documentation_mdbook.md @@ -1,6 +1,8 @@ # This Documentation -This documentation is generated using mdbook. you can check out the [mdbook documentation](https://rust-lang.github.io/mdBook/) for more info on mdbook. +This documentation is created using mdbook and aims to provide a concise reference for users and developers of youki. For more information on mdbook itself, you can check out the [mdbook documentation](https://rust-lang.github.io/mdBook/). + +Please make sure that you update this documentation along with newly added features and resources that you found helpful while developing, so that it will be helpful for newcomers. TODO, update after deciding the final way to host the mdbook To compile and deploy this mdbook follow the instructions diff --git a/docs/src/developer/good_places_to_start.md b/docs/src/developer/good_places_to_start.md index 5caecfc92..b16474b5a 100644 --- a/docs/src/developer/good_places_to_start.md +++ b/docs/src/developer/good_places_to_start.md @@ -2,22 +2,22 @@ First of all, welcome to youki! Hope you have fun while developing and contributing :) -This lists some of the known places that are long-running and would be useful for beginners. But as the things under development can change any time, the best place to check for is the issues on [GitHub repo](https://github.com/containers/youki/issues). You can find issues with labels `good fist issue` or `help wanted` and start working on them. +This lists some of the known places that are long-running and would be useful for beginners. But as the things under development can change any time, the best place to check are the issues on the [GitHub repo](https://github.com/containers/youki/issues). You can find issues with labels `good fist issue` or `help wanted` and start working on them. -Other that that you can search for `TODO` or `FIXME` comments in the source, and try working on them, but some of them can be hard issues. +You can also search for `TODO` or `FIXME` comments in the source, and try working on them, but not all of them are easy places to start, can some of them can be particularly tricky to fix. --- -This lists known sections that can be good for beginners at the time of the writing. Please update as things change. +This lists known parts of youki that can be good for beginners at the time of the writing. Please update as things change. #### Documentation Comments Currently youki is decently commented, and those explain most of the public facing API and structs. But there are still places which can use more doc comments, and examples showing usage, so people can use the docs generated by `cargo doc` as a guide. -If you don't know much about container runtimes or low level system working, then this can be a good place to start, as while going through the code an documenting it, you can learn about it. Make sure that you update this documentation with useful links that you found while commenting some code if it has some peculiar behavior. +If you don't know much about container runtime or low level system working, then this can be a good place to start. While going through the code an documenting it, you can learn about it. Make sure that you update this documentation with useful links that you found while commenting some code if it has some peculiar behavior, or it is hard to understand without knowing some background. #### Integration Tests -You can find more detailed information about this in the integration_test crate, but in brief, we currently use [OCI-runtime-tools](https://github.com/opencontainers/runtime-tools) provided integration tests to validate that youki performs as per the OCI spec. But those are written in Go, which makes the developer depend on two language env to compile youki and test it ; as well as those have some issue which make it hard to run them on some local systems. +You can find more detailed information about this in the integration_test crate, but in brief, we currently use [OCI-runtime-tools](https://github.com/opencontainers/runtime-tools) provided integration tests to validate that youki is OCI spec compliant. But those are written in Go, which makes the developer depend on two language env to compile youki and test it. These tests also have some issues which makes them hard to use on some system setups. -Thus we are porting those test to Rust, so that it can be an Rust implementation of OCI-runtime integration tests, and easy to run on locacl systems. If you know Go and Rust this can be a great place to start. Check out the [tracking issue](https://github.com/containers/youki/issues/361). +Thus we are porting those test to Rust, so that it can be an Rust implementation of OCI-runtime integration tests, as well as be easy to run on local systems for testing. If you know Go and Rust this can be a great place to start. Check out the [tracking issue](https://github.com/containers/youki/issues/361). diff --git a/docs/src/developer/integration_test.md b/docs/src/developer/integration_test.md index fe83183e4..dc5a4c4de 100644 --- a/docs/src/developer/integration_test.md +++ b/docs/src/developer/integration_test.md @@ -1,5 +1,9 @@ # integration_test -This crate is Rust port of OCI-runtime tools validation integration tests. This is still a work in progress, and you can check the [tracking issue](https://github.com/containers/youki/issues/361) for the current progress. +This crate contains the Rust port of OCI-runtime tools integration tests, which are used to test if the runtime works as per the OCI spec or not. Initially youki used the original implementation of these test provided in the OCI repository [here](https://github.com/opencontainers/runtime-tools/tree/master/validation). But those tests are written in Go, which made the developers depend on two language environments Rust and Go to compile youki and test it. The Validation tests themselves also have an optional dependency on node js to parse their output, which can make it a third language dependency. -TODO add explanation of reasons behind this, and a brief explanation of the crate implementation +Other than that, those tests also showed some issues while running on some local systems, and thus running the tests would be difficult on local system. As the runtime is a complex piece of software, it becomes useful to have a set of tests that can be run with changes in code, so one can verify that change in one part of youki has not accidentally broken some other part of youki. + +Thus we decided to port the tests to Rust, and validate them, so that we have a set of unit tests as well of integration tests to validate the working of runtime. These tests are still under development, and you can check the [tracking issue](https://github.com/containers/youki/issues/361) for more details. More details on working of these tests can be found at [https://github.com/containers/youki/tree/main/crates/integration_test](https://github.com/containers/youki/tree/main/crates/integration_test). + +As these tests are under development, these are validated on a standard runtime such as runc in the GitHub CI, so validate the tests themselves. diff --git a/docs/src/developer/introduction.md b/docs/src/developer/introduction.md index 66dc81b55..4538e081f 100644 --- a/docs/src/developer/introduction.md +++ b/docs/src/developer/introduction.md @@ -1,15 +1,17 @@ # Developer Documentation -This is the documentation of various parts of youki for those who wish to contribute in the development of youki. +This section of the documentation is more oriented towards those who wish to contribute to youki, and contains more information and resources about the working and implementation of it. So if you are thinking of helping, this is a great place to start with. -First of all, Thank you! If you have any issues or doubts, you can ask them on youki's discord server [here](https://discord.gg/h7R3HgWUct). +Also, Thank you! If you have any issues or doubts, you can ask them on youki's discord server [here](https://discord.gg/h7R3HgWUct). -This is split into pars as below : +This section is split into following parts -- Basics : contains resources and information that you will probably need for modifying any part of the code +- Basics : This contains general resources and information that you wold need to work with any parts of youki. -- Unwritten Rules : This is the section to write down rules or conventions that are decided in PR discussions +- Unwritten Rules : This is the part to make them written! This will contain conventions and rules that were discussed and decided in PRs or just commonly followed when developing. -- Good Places to Start : This will give you some easy-entry points to start contributing +- Good Places to Start : This section will contain some suggestions about the areas that will be a good place to start for new contributors. -- Crate specific Information : This sections contains sub-sections for each crate in youki workspace, and will have resources specific to that crate +- Crate specific Information : This is split into one sections for each crate, and will contains information and resources specific for that crate. + +Happy Contributing! diff --git a/docs/src/developer/libcgroups.md b/docs/src/developer/libcgroups.md index 5271a8466..dc26c4259 100644 --- a/docs/src/developer/libcgroups.md +++ b/docs/src/developer/libcgroups.md @@ -1,9 +1,30 @@ # libcgroups -This crate provides and interface for working with cgroups in Linux. cgroups or control groups are Linux kernel feature which can be used to fine-control resources and permissions given to a particular process or a group of processes. You can read more about them on the [cgroups man page](https://man7.org/linux/man-pages/man7/cgroups.7.html). +This crate provides an interface for working with cgroups in Linux. cgroups or control groups is a Linux kernel feature which can be used to fine-control resources and permissions given to a particular process or a group of processes. You can read more about them on the [cgroups man page](https://man7.org/linux/man-pages/man7/cgroups.7.html). -The initial version of cgroups is called the version 1 was implemented in kernel 2.6.24 , later in kernel version 4.5, a new version of cgroups was released, aimed to solve issues with v1, and was the version v2. +The initial version of cgroups is called the version 1 was implemented in kernel 2.6.24, and later in kernel version 4.5, a new version of cgroups was released, aimed to solve issues with v1, the version v2. -This crate exposes modules to work with both of the version, as well as structs needed to store cgroups data. Apart from these two modules, cgroups crate also exposes utility functions to read cgroups files and parse their data, and a stats module that exposes the structs and functions to parse and store the statistics related to cgroups, such as cpu and memory usage, Huge TLB size and hits etc. +This crates exposes several functions and modules that can be used to work with cgroups : + +- common traits and functions which are used by both v1 and v2 such as + + - Trait CgroupManager, this abstracts over the underlying implementation of interacting with specific version of cgroups, and gives functions to add certain process to a certain cgroup, apply resource restrictions, get statistics of a cgroups, freeze a cgroup, remove a cgroup or get list of all processes belonging to a cgroup. v1 and v2 modules both contain a version specific cgroup manager which implements this trait, and thus either can be given to functions or structs which expects a cgroup manager, depending on which cgroups the host system uses. + - Apart from the trait, this also contians functions which help with reading cgroups files, and write data to a cgroup file, which are used throughout this crate. + - A function to detect which cgroup setup (v1, v2 or hybrid) is on the host system, as well as a function to get the corresponding cgroups manager. + +- Functions and structs to get and store the statistics of a cgroups such as + + - CPU stats including usage and throttling + - Memory stats including usage of normal and swap memory, usage of kernel memory, page cache in bytes etc + - Pid stat including current active pids nd maximum allowed pids + - Block IO stats such as number of bytest transferred to/from a device in the cgroup, io operations performed by a device in the cgroup, amount of time cgroup had access to a device etc + - Huge TLB stats such as usage and maximum usage etc. + - Function to get pid stats + - Function to get supported hugepage size + - Function to parse flat keyed data and nested keyed data that can be in a cgroups file + - Parse a device number + +- Cgroups V1 module which deal with implementing a cgroup manager for systems which have cgroups v1 or hybrid cgroups +- Cgroups V2 module which deal with implementing a cgroup manager for systems which have cgroups v2 As youki currently depends on systemd as an init system, this crate also exposes module systemd, which provides interface for working with systemd related operations. [systemd resource control](https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html) is a good place to read more about systemd and its involvement in resource control. diff --git a/docs/src/developer/libcontainer.md b/docs/src/developer/libcontainer.md index ae2c5456f..d22bdb7f8 100644 --- a/docs/src/developer/libcontainer.md +++ b/docs/src/developer/libcontainer.md @@ -1,36 +1,51 @@ # libcontainer -This is one of the core crates part of the youki workspace. This deals with the actual creation and management of the container processes, and provides functions and structs for the same. +THis crate is one of the core crates of the youki workspace, and has functions and structs that deal with the actual craetion and management of the container processes. -Remember, that in the end, a container is just another process in Linux, which has control groups, namespaces, pivot_root and other mechanisms applied to it. The program executing has the impression that is is running on a complete system, but from the host system's perspective, it is just another process, and has attributes such as pid, file descriptors, etc. associated with it like any other process. +Remember, in the end, a container is just another process in Linux, which has control groups, namespaces, pivot_root and other mechanisms applied to it. The program executing has the impression that is is running on a complete system, but from the host system's perspective, it is just another process, and has attributes such as pid, file descriptors, etc. associated with it like any other process. -When given the create command, Youki will load the specification, configuration, sockets etc., use clone syscall to create the container process (init process), applies the limits, namespaces, and etc. to the cloned container process. The container process will wait on a unix domain socket before executing the command/program. +Along with the container related functions, this crate also provides Youki Config, a subset of the OCI spec config. This config contains only the essential data required for running the containers, and due to its smaller size, parsing it and passing it around is more efficient than the complete OCI spec config struct. -The main youki process will setup pipes to communicate and synchronize with the intermediate and init process. The init process will notify the intermediate process, and then intermediate process to the main youki process that it is ready and start to wait on a unix domain socket. The youki process will then write the container state and exit. +Other than that, this crate also provides a wrapper over basic Linux sockets, which are then used internally as well as by youki to communicate between the main youki process and the forked container process as well as the intermediate process. -The following are some resources that can help understand with various Linux features used in the code of this crate. +This crate also provides an interface for Apparmor which is another Linux Kernel module allowing to apply security profiles on a per-program basis. More information about it can be found at [https://apparmor.net/](https://apparmor.net/). -- [mio Token definition](https://docs.rs/mio/0.7.11/mio/struct.Token.html) -- [oom-score-adj](https://dev.to/rrampage/surviving-the-linux-oom-killer-2ki9) -- [unshare man page](https://man7.org/linux/man-pages/man1/unshare.1.html) -- [user-namespace man page](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) -- [wait man page](https://man7.org/linux/man-pages/man3/wait.3p.html) +### Notes -The main youki process creates the intermediate process and the intermediate process creates the container process (init process). The hierarchy is: `main youki process -> intermediate process -> init process` +#### Some other modules expose by this crate are -The main youki process will set up pipes used as message passing and synchronization mechanism with the init process. The reason youki needs to create/fork two process instead of one is due to the user and pid namespaces. In rootless container, we need to first enter user namespace, since all other namespaces requires CAP_SYSADMIN. When unshare or set_ns into pid namespace, only the children of the current process will enter into a different pid namespace. As a result, we must first fork a process to enter into user namespace, call unshare or set_ns for pid namespace, then fork again to enter into the correct pid namespace. +- rootfs, which is a ramfs like simple filesystem used by kernel during initialization +- hooks, which allow running of specified program at certain points in the container lifecycle, such as before and after creation, start etc. +- singals, which provide a wrapper to convert to and from signal numbers and text representation of signal names +- capabilities, which has functions related to set and reset specific capabilities, as well as to drop extra privileges + - [Simple explanation of capabilities](https://blog.container-solutions.com/linux-capabilities-in-practice) + - [man page for capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) +- tty module which daels with providing terminal interface to the container process + - [pseudoterminal man page](https://man7.org/linux/man-pages/man7/pty.7.html) : Information about the pseudoterminal system, useful to understand console_socket parameter in create subcommand + +#### Namespaces : namespaces provide isolation of resources such as filesystem, process ids networks etc on kernel level. This module contains structs and functions related to applying or un-applying namespaces to the calling process. + +- [pid namespace man page](https://man7.org/linux/man-pages/man7/pid_namespaces.7.html) +- [CLONE_NEWUSER flag](https://man7.org/linux/man-pages/man2/clone.2.html) Note: clone(2) offers us the ability to enter into user and pid namespace by creatng only one process. However, clone(2) can only create new pid namespace, but cannot enter into existing pid namespaces. Therefore, to enter into existing pid namespaces, we would need to fork twice. Currently, there is no getting around this limitation. - [fork(2) man page](https://man7.org/linux/man-pages/man2/fork.2.html) - [clone(2) man page](https://man7.org/linux/man-pages/man2/clone.2.html) -- [pid namespace man page](https://man7.org/linux/man-pages/man7/pid_namespaces.7.html) -This has functions related to set and reset specific capabilities, as well as to drop extra privileges +#### Pausing and resuming -- [Simple explanation of capabilities](https://blog.container-solutions.com/linux-capabilities-in-practice) -- [man page for capabilities](https://man7.org/linux/man-pages/man7/capabilities.7.html) +Pausing a container indicates suspending all processes in it. This can be done with signals SIGSTOP and SIGCONT, but these can be intercepted. Using cgroups to suspend and resume processes without letting tasks know. -This has functions related to setting of namespaces to the calling process +- [cgroups man page](https://man7.org/linux/man-pages/man7/cgroups.7.html) +- [freezer cgroup kernel documentation](https://www.kernel.org/doc/Documentation/cgroup-v1/freezer-subsystem.txt) -- [CLONE_NEWUSER flag](https://man7.org/linux/man-pages/man2/clone.2.html) +#### The following are some resources that can help understand with various Linux features used in the code of this crate. + +- [oom-score-adj](https://dev.to/rrampage/surviving-the-linux-oom-killer-2ki9) +- [unshare man page](https://man7.org/linux/man-pages/man1/unshare.1.html) +- [user-namespace man page](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) +- [wait man page](https://man7.org/linux/man-pages/man3/wait.3p.html) +- [pipe2 man page](https://man7.org/linux/man-pages/man2/pipe.2.html) : Definition and usage of pipe2 +- [Unix Sockets man page](https://man7.org/linux/man-pages/man7/unix.7.html) : Useful to understand sockets +- [prctl man page](https://man7.org/linux/man-pages/man2/prctl.2.html) : Process control man pages diff --git a/docs/src/developer/liboci_cli.md b/docs/src/developer/liboci_cli.md index 14eca6ebe..49796e72d 100644 --- a/docs/src/developer/liboci_cli.md +++ b/docs/src/developer/liboci_cli.md @@ -1,5 +1,7 @@ # liboci-cli -This crate was separated from original youki crate, and now contains a standalone implementation of structs needed for parsing commandline arguments for OCI-spec compliant runtime commandline interface. +This crate was separated from original youki crate, and now contains a standalone implementation of structs needed for parsing commandline arguments for OCI-spec compliant runtime commandline interface. This is in turn used by youki to parse the command line arguments passed to it, but can be used in any other projects where there is need to parse OCI spec based commandline arguments. This primarily uses the crate clap-v3 for parsing the actual commandline arguments given to the runtime. + +You can refer to [OCI Commandline interface guide](https://github.com/opencontainers/runtime-tools/blob/master/docs/command-line-interface.md) to know more about the exact specification. diff --git a/docs/src/developer/libseccomp.md b/docs/src/developer/libseccomp.md index 6807e2a92..fdb3273c9 100644 --- a/docs/src/developer/libseccomp.md +++ b/docs/src/developer/libseccomp.md @@ -1,5 +1,5 @@ # libseccomp -Seccomp is a linux kernel feature that allows a process a one-way transition into secure mode, where restrictions are applied to the syscalls the process can make, as well as restrictions on the file descriptors. +Seccomp is a linux kernel feature that allows a process a one-way transition into secure mode, where restrictions are applied to the syscalls the process can make, as well as restrictions on the file descriptors. Specifically, it can exit, sigreturn and read/write already open file descriptors. This way the process can be isolated and restricted on how it interacts with rest of the system on a kernel level. -This crate does not actually implement any particular feature, but provides Rust FFI bindings for seccomp module. These are primarily generated by using rsut-bindgen on seccomp C header file, and then manually fixed where any issues were found. +This crate does not actually implement any particular feature, but provides Rust FFI bindings for seccomp module. These are primarily generated by using rsut-bindgen on seccomp C header file, and then manually fixed where any issues were found. More information about seccomp can be found in its [man page](https://man7.org/linux/man-pages/man2/seccomp.2.html). diff --git a/docs/src/developer/test_framework.md b/docs/src/developer/test_framework.md index a20dc4ca8..513700b72 100644 --- a/docs/src/developer/test_framework.md +++ b/docs/src/developer/test_framework.md @@ -1,5 +1,7 @@ # test_framework -This is the framework specifically developed to implement the ported integration tests. This exposes Structs to represent individual tests, test groups and test managers which runs the test groups. This also exposes a trait which can be used to implement a custom test struct or a custom test group. +This crate contains the testing framework specifically developed for porting the OCI integration test to rust. This contains structs to represent the individual tests, group of tests and a test manager that has responsibility to run tests. This Also exposes traits which can be used to implement custom test structs or test group structs if needed. -By default the test groups are run in parallel using [crossbeam crate](https://www.crates.io/crates/crossbeam), and the default test_group implementation also runs individual tests parallelly. Sometimes you might need to run the individual test in certain order, serially such as when testing container lifecycle. In such cases you will need to implement the TestableGroup trait in a custom struct so you can finely control the order of execution. +By default the test groups are run in parallel using the [crossbeam crate](https://www.crates.io/crates/crossbeam), and the default test_group implementation also runs individual tests parallelly. + +Sometimes you might need to run the tests in a test group serially or in certain order, for example in case of testing container lifecycle, a container must be created and started before stopping it. In such cases, you will need to implement the respective traits on your own structs, so that you can have fine control over thr running of tests. Check the readme of the test_framework crate to see the struct and trait documentation [here](https://github.com/containers/youki/tree/main/crates/test_framework). diff --git a/docs/src/developer/unwritten_rules.md b/docs/src/developer/unwritten_rules.md index 33b0ae307..024f50b06 100644 --- a/docs/src/developer/unwritten_rules.md +++ b/docs/src/developer/unwritten_rules.md @@ -1,18 +1,18 @@ # Unwritten Rule -This is the place to write down rules or conventions that are discussed in PRs, so that newcomers can easily find them, without having to go through the PR history. So if you decide on any convention to follow for the project, please make sure to add them here. +This is the place to write down rules or conventions that were discussed in PRs, so that newcomers can easily find them, without having to go through the PR history. So if you decide on any convention to follow for the project, please make sure to add them here. ## Conventions to follow #### Errors -Youki currently uses [anyhow](https://www.crates.io/crates/anyhow) library to deal with errors occurring while its execution. So wherever you use fallible actions, or functions that can return `Result`, make sure you attach enough information with the errors so that error logs can be useful for debugging later. For example, if you are reading a file, or parsing something and the operation does not succeed, you can add the path from which you attempted to read the file, or the string that you attempted to parse. +Youki currently uses [anyhow](https://www.crates.io/crates/anyhow) library to deal with errors occurring during its execution. So wherever you use fallible actions, or functions that can return `Result`, make sure you attach enough information with the errors so that error logs can be useful for debugging later. For example, if you are reading a file, or parsing something and the operation does not succeed, you can add the path from which you attempted to read the file, or the string that you attempted to parse. Also for the error messages, we follow the convention all small-case letters and no period at the end, as discussed in [this PR](https://github.com/containers/youki/issues/313). Whenever you write error messages, please follow this convention to keep them uniform. #### Logs -youki uses [log](https://crates.io/crates/log) crate to log information while running. Whenever adding code to interact with system or kernel faetures or such, make sure to add debug logs so that if youki crashes, you can trace the errors using logs. +youki uses [log](https://crates.io/crates/log) crate to log information while running. Whenever adding code to interact with system or kernel features or such, make sure to add debug logs so that if youki crashes, you can trace the errors and zero-in on the reasons using logs. #### Comments @@ -20,4 +20,4 @@ Make sure that you comment copiously, and explain the peculiar behavior of your #### Update This Documentation -Keep this Documentation updated ! Make sure you add any relevant doc-links and resources to this as you add code, so that it can help newcomers as well as others to find the resources in one single place. +Keep this Documentation updated! Make sure you add any relevant doc-links and resources to this that you found helpful or contains background information required to understand the code, so that it can help newcomers as well as others to find the resources in one single place. diff --git a/docs/src/developer/youki.md b/docs/src/developer/youki.md index ad3ea3f1c..e3432b38a 100644 --- a/docs/src/developer/youki.md +++ b/docs/src/developer/youki.md @@ -1,9 +1,23 @@ # youki -This is the core crate that contains the youki binary itself. The simple control flow of youki can be explained as : +This is the core crate that contains the youki binary itself. This provides the user interface, as well as binds the ther crates together to actually perform the work of creation and management of containers. THus this provides implementation of all the commands supported by youki. + +The simple control flow of youki can be explained as :

-TODO Add explanation here +When given the create command, Youki will load the specification, configuration, sockets etc., and use clone syscall to create an intermediate process. This process will set the cgroups and capabilities, and then fork to the init process. Reason to create this intermediate prcoess is that the clone syscall cannot enter into existing pid namespace that has been created for the container. Thus first we need to make a transition to that namespace in the intermediate process and fork that to the container process. After that the main youki process is requested the uid and gid mappings, and after receiving them the intermediate process sets these mapping, fork the init process and return pid of this init process to the main youki process before exiting. + +THe init process then transition completely into the new namespace setup for the container (the init process only transitions the pid namespace). It changes the root mountpoint for the process using [pivot_root](https://man7.org/linux/man-pages/man2/pivot_root.2.html), so that the container process can get impression that it has a complete root path access. After that the init process sets up the capabilities and seccomp, and sends the seccomp notify fd to the main youki process. When the seccomp agent running on the host system sets up the seccomp profile, it notifies the init process, after which it can execute the programto be executed inside the container. Thus the init process then sends ready notification to the main youki process, and waits for the start signal. + +The main youki process which started creating the container, when receives the ready signals update the pid file of the container process and exits. This concludes the creation of the container. + +To start the container, when youki start it executed along with the container id, start signal is sent to the waiting container init process, and the the youki process exists. + +When the init process receives the start signal, it execs the program to be run in the container, and then exits. + +### Notes + +The main youki process will set up pipes used as message passing and synchronization mechanism with the init process. The reason youki needs to create/fork two process instead of one is due to the user and pid namespaces. In rootless container, we need to first enter user namespace, since all other namespaces requires CAP_SYSADMIN. When unshare or set_ns into pid namespace, only the children of the current process will enter into a different pid namespace. As a result, we must first fork a process to enter into user namespace, call unshare or set_ns for pid namespace, then fork again to enter into the correct pid namespace. From ecb8bb8c0331a79dac342943e4775e6af709a10f Mon Sep 17 00:00:00 2001 From: Yashodhan Joshi Date: Fri, 24 Dec 2021 20:38:23 +0530 Subject: [PATCH 5/5] Add deploy CI script --- .github/workflows/docs.yaml | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/docs.yaml diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 000000000..90ae0433b --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,38 @@ +on: + push: + branches: + - main + +jobs: + changes: + runs-on: ubuntu-latest + outputs: + dirs: ${{ steps.filter.outputs.changes }} + steps: + - uses: actions/checkout@v2 + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + docs: docs/** + deploy: + needs: [changes] + if: ${{ !contains(needs.changes.outputs.dirs, '[]') }} + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + steps: + - uses: actions/checkout@v2 + - name: Setup mdBook + uses: peaceiris/actions-mdbook@v1 + with: + mdbook-version: 'latest' + - name: Build mdbook + working-directory: ./docs + run: mdbook build + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.ref == 'refs/heads/main' }} + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/book