From a48c5c0f80abb73a6c96ee96898b19c105c3f045 Mon Sep 17 00:00:00 2001 From: weizhichen Date: Thu, 9 Jun 2022 09:14:11 +0000 Subject: [PATCH] change blobfuse-proxy from daemonset to initContainer feat: upgrade to blobfuse v1.4.4 shellcheck shellcheck fix helm rm /tmp fix : 1. do not RESTART blobfuse-proxy 2. move dockerfile from build dir to pkg dir remove Restart=always from service file chore: combine case insensitive key/value setting in parameters fix feat: add runOnControlPlane in chart config Update driver-parameters.md Update driver-parameters.md generate tgz file for latest helm chart move blobfuse-proxy files to pkg dir fix fix test --- .github/workflows/linux.yaml | 1 - .gitignore | 2 - Makefile | 5 +- README.md | 2 +- charts/README.md | 8 +- charts/latest/blob-csi-driver-v1.14.0.tgz | Bin 5977 -> 5405 bytes .../templates/blobfuse-proxy.yaml | 126 ------------------ .../templates/csi-blob-controller.yaml | 3 + .../templates/csi-blob-node.yaml | 43 ++++++ charts/latest/blob-csi-driver/values.yaml | 5 +- deploy/blobfuse-proxy.yaml | 115 ---------------- deploy/csi-blob-node.yaml | 33 +++++ deploy/install-driver.sh | 1 - deploy/uninstall-driver.sh | 6 - docs/driver-parameters.md | 9 +- docs/install-csi-driver-master.md | 4 +- pkg/blob/blob.go | 15 +++ pkg/blob/blob_test.go | 54 ++++++++ pkg/blob/controllerserver.go | 4 +- pkg/blob/nodeserver.go | 6 +- .../system => }/blobfuse-proxy.service | 0 pkg/blobfuse-proxy/debpackage/DEBIAN/control | 7 - pkg/blobfuse-proxy/init.sh | 67 ++++++++++ pkg/blobfuse-proxy/rpmbuild/SPECS/utils.spec | 25 ---- pkg/blobplugin/Dockerfile | 12 +- 25 files changed, 244 insertions(+), 309 deletions(-) delete mode 100644 charts/latest/blob-csi-driver/templates/blobfuse-proxy.yaml delete mode 100644 deploy/blobfuse-proxy.yaml rename pkg/blobfuse-proxy/{debpackage/etc/systemd/system => }/blobfuse-proxy.service (100%) delete mode 100644 pkg/blobfuse-proxy/debpackage/DEBIAN/control create mode 100644 pkg/blobfuse-proxy/init.sh delete mode 100644 pkg/blobfuse-proxy/rpmbuild/SPECS/utils.spec diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index f54ca3c3d..1cebf58ea 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -41,7 +41,6 @@ jobs: run: | export PATH=$PATH:$HOME/.local/bin make blobfuse-proxy - sudo dpkg -i _output/blobfuse-proxy.deb - name: Get code coverage env: diff --git a/.gitignore b/.gitignore index 303cb669d..64ddbb83a 100644 --- a/.gitignore +++ b/.gitignore @@ -69,5 +69,3 @@ cscope.* /bazel-* *.pyc profile.cov - -pkg/blobfuse-proxy/debpackage/usr/ diff --git a/Makefile b/Makefile index 1aff16850..94f8659d0 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,6 @@ e2e-bootstrap: install-helm docker pull $(IMAGE_TAG) || make blob-container push helm install blob-csi-driver ./charts/latest/blob-csi-driver --namespace kube-system --wait --timeout=15m -v=5 --debug \ --set controller.replicas=1 \ - --set controller.runOnMaster=true \ --set cloud=$(CLOUD) \ $(E2E_HELM_OPTIONS) @@ -179,6 +178,4 @@ delete-metrics-svc: .PHONY: blobfuse-proxy blobfuse-proxy: - mkdir -p ./pkg/blobfuse-proxy/debpackage/usr/bin/ ./_output - CGO_ENABLED=0 GOOS=linux go build -mod vendor -ldflags="-s -w" -o ./pkg/blobfuse-proxy/debpackage/usr/bin/blobfuse-proxy ./pkg/blobfuse-proxy - $(DPKG_DEB) --build pkg/blobfuse-proxy/debpackage ./_output/blobfuse-proxy.deb + CGO_ENABLED=0 GOOS=linux go build -mod vendor -ldflags="-s -w" -o _output/${ARCH}/blobfuse-proxy ./pkg/blobfuse-proxy diff --git a/README.md b/README.md index 5ecc45740..27bf9ebf4 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ This driver allows Kubernetes to access Azure Storage through one of following m ### Container Images & Kubernetes Compatibility: |driver version |Image | supported k8s version | built-in blobfuse version | |----------------|-------------------------------------------|-----------------------|---------------------------| -|master branch |mcr.microsoft.com/k8s/csi/blob-csi:latest | 1.20+ | 1.4.3 | +|master branch |mcr.microsoft.com/k8s/csi/blob-csi:latest | 1.20+ | 1.4.4 | |v1.13.0 |mcr.microsoft.com/k8s/csi/blob-csi:v1.13.0 | 1.20+ | 1.4.3 | |v1.12.0 |mcr.microsoft.com/k8s/csi/blob-csi:v1.12.0 | 1.20+ | 1.4.3 | |v1.11.0 |mcr.microsoft.com/k8s/csi/blob-csi:v1.11.0 | 1.20+ | 1.4.3 | diff --git a/charts/README.md b/charts/README.md index fdf81bba8..d44a25f5b 100644 --- a/charts/README.md +++ b/charts/README.md @@ -6,8 +6,7 @@ ### Tips - configure with [blobfuse-proxy](../deploy/blobfuse-proxy) to make blobfuse mount still available after driver restart - specify `node.enableBlobfuseProxy=true` together with [blobfuse-proxy](../deploy/blobfuse-proxy) - - make controller only run on master node: `--set controller.runOnMaster=true` - - enable `fsGroupPolicy` on a k8s 1.20+ cluster: `--set feature.enableFSGroupPolicy=true` + - run controller on control plane node: `--set controller.runOnControlPlane=true` - set replica of controller as `1`: `--set controller.replicas=1` - specify different cloud config secret for the driver: - `--set controller.cloudConfigSecretName` @@ -92,11 +91,12 @@ The following table lists the configurable parameters of the latest Azure Blob S | `controller.cloudConfigSecretName` | cloud config secret name of controller driver | `azure-cloud-provider` | `controller.cloudConfigSecretNamespace` | cloud config secret namespace of controller driver | `kube-system` | `controller.allowEmptyCloudConfig` | Whether allow running controller driver without cloud config | `true` -| `controller.replicas` | the replicas of csi-blob-controller | `2` | +| `controller.replicas` | replica number of csi-blob-controller | `2` | | `controller.hostNetwork` | `hostNetwork` setting on controller driver(could be disabled if controller does not depend on MSI setting) | `true` | `true`, `false` | `controller.metricsPort` | metrics port of csi-blob-controller | `29634` | | `controller.livenessProbe.healthPort ` | health check port for liveness probe | `29632` | -| `controller.runOnMaster` | run controller on master node | `true` | +| `controller.runOnMaster` | run controller on master node | `false` | +| `controller.runOnControlPlane` | run controller on control plane node |`false` | | `controller.logLevel` | controller driver log level | `5` | | `controller.resources.csiProvisioner.limits.memory` | csi-provisioner memory limits | 100Mi | | `controller.resources.csiProvisioner.requests.cpu` | csi-provisioner cpu requests | 10m | diff --git a/charts/latest/blob-csi-driver-v1.14.0.tgz b/charts/latest/blob-csi-driver-v1.14.0.tgz index 76f6effc4518076cd8a5e1349d550f43efc08deb..92c68523ea424d93f67fbec9cd8e541d083e0cc9 100644 GIT binary patch delta 5395 zcmV+u73}KSE}bfnJAZv!bK5ww@I32Rpp@C#+5JLYe4SO5AAC`=t~jzvanCpZUa{chL2?$7G*_0IDioLHeA{h|&+hEeMzp&4*|NoPq?QP$+y( zn0880&JG*xMr##HpUi-Q5Pi-@upY*wOQ^bJ@EtTxR)6E8%!4Z}TLYL7Avu1*I92CZ zG98|D&rxu^ngUml0SNN@OD33`4jEMK#?dNNHP>g67jF{%Xyg<3Qhlk_-n>Br7y}3D z0+I|d3W#wOFb+t7%RvhDc?2jD;Gr*5pc)OK6Qns*C=eXR9W<3P6rE;<=mud;kGY~L zv3L+omw(YQnM53%ka>>FsKkT2IhpE%&>%HO6Sd*{@#y#e`u)bpkIEiy+IB# zTUs!W7jOOn=B7yx2T-dC;4|XFx$ClsN;SVO2P~n3B#!{SLH|-iQ9r*$aa)8y)+fXG;VL|-t~AAchO>UWOV7urzx7?qs)KJboOX~&4- z;-e!ldV>_Ph>g)OCLyLbp7R7h<9hmIrpNm|3*z@#%zK`S-P{=${3z>JGw(6j#=Q0={kHB9B;YK$wosr|9%ScET6#5Y= z8Gi~uVh>APb9IwUM}*#>ho6Om!?0}kD4K3y2HHx(sLu)GM9wdLEHr$eLK#zfF%?lr zM0`eOfH(*iaTk!7GA`HUewR@fLYWGHo}H!=f-pwygd8Byrd+cYX6fna6%lhGA#hwj z5)!!Maw3U~eT97pZlqdkeuGDeCW=xupMNY28J7`Nm69PIQM3>`gax9Ea>gVY&5-~p zl}KYH^cviyfu=Bw<_I2f5c+^R>i0sE4ti3s&&4O9SiRk9wb65}+~~Ph%ZS8+*({== z_)@8-OFdtyRCf?o3e3@!W~=e4TZ<68&_QZJ95a74qe=?0;i-9?`T`%jsWJ%5m2 z7=Kvk2{|QVF+VPgd`}6KM+6@E;3*}L5{^6uYF;JsAOz|mM$yONb<(|t6vuMniaz7A zgAV?2a(HCl`KEwtIE2t>A(8%Bx?f>O7Gq?cv9)yUef zM3x#z)i@4biKDoYf32y*<1ta<9IH5`{fD4bE93De$$a3k_SH}v^Z@e?s_UJCGanjx z4Hak(s>c{vsCp}C_3k4PQX~~H9)l}8sNQ2kcM4wQLwzmyv2cWn8NWIf=YJJwYIJN zLoW(4-GV|z{U7E`XBIZr$$y_h4@LyntqfrB8MfHIPLyT`u-7G?08v~XR*Q;Py_TBJ3 zXHl4(jgqxBmG40s*&0f$&KIV(W^Bby&M0oUf;6uSb3Ui83XG;%{%!Rrli4`?u)2 z_wM#B`q%lOcinsc7Ig=M+d-}76B<1#VwHCBL3U7chI!K`qh|c2rUgWD)tK2GanYkX2{)M|GOxl?c-Rsb{ST#02Q-lEV4EI^$mI<;7*xY9?CLlZOR?CA+L z2B6zD7RC*FepVzIV2&eV?R{3~re&!YdVa1W$N7S#JKgJ}Tz{#S|NANUA#l-<;VVM{ zE9C#+uoVA0IX*qu%l|IQ(^K>BXhwpLvWU@`_#o#YoCRuH+$nU>-oyS`2lH)Hw8rurQA0(LANtc={SOJK?%X ziY(-W$}y_{ML2&E_51+L&+!=p{h=KwN=w!MUmex&>zfRQ1p%WdQEZ}$$fSYsNE&E3 z3=5}LV^}^tp+vTGYtum1PWn(qUDL6sOsh#Y-qR|2{eKLds*+IGkG!hUR8Z!!AY@dV z!5L?3xm}34blCXCBM&ge6Y!kTJU{lt<(<$zPFDVW5i_>~T*SW>v{IKp&~FiwxmkUF zt|=?NA*M~uhPENZ{MpEo6H8LoOKqx=WdFf|zof+iac*Ue*oTqtFQPPK##8sMK=ySG z(?CjQv42wvi_sQ^xD5KuaOyys3+Y&)Y*xbdYu=Rxw&u zHq-E9hLM&>nPT^7Zf>f%x`-?r)?5Q+E+kZec^T7XC#+zstztAw06()>rP`vUSw&IC z`<>#-xx`;UV`?g<+~8zlU)e;icwwek^nW?%15&e|7lcMWfZRwknX+momaGn{XQspS zPmYB6-$6XbDR-JW7rolTzd)JAdSv z>dVD3Y41^b!Aw^fUiCduX3Wu{|5A9Vs)kv4O(S|4`n%Y!=tn^=rLl?HGNk)2&dPIx;*|hk*P#1D0nalhRy1VRe zLPnlb~@9z5i=KkIFP1j}= zzT+$?j#VFQ#$@slhaX@*fN{A&vMh1x9tY1E6CW(?bk3ogJxd+a$$#3|&t#_+2a;8i zcB3|;>SK zGt{^4o3X;R`bd(NoBL#5iEBOccwMiH)n~H{Pt0GPDW;wej(<^L4xy0%2RLW^OlPnD z_1n5zlDYrh&zdQz72;+HwRlLa^l8IbCyzgs<<+X9K7^SO~|}!I%c<2Z|@5$xu6*18#)ck z^&CyySmZA3yff$XY{=Mb%{OEW?%eg^BlBQ!C#bLHOn*r8$b$ON4W|$Q$Nry4Nv(K0Q8e?d5+LCC>lC5HLD~4bs0fcWg89s{n8#wGrvdnM293 zgh_l&RN3W(PRzZ{#1t#uWG&K>uTLm4cJUw!aFfij(n!Ba&X|RYeYRM_NS1-^HleG4 zTrZa8f`4>A_p5(>6>o{lQ^PX8C69_^8@w{NG!|!wm*j*g`Zmo5D{=#OtVhSqB|t-j^SiRe3+D#>OURXT9SwZb+(G*jo@?-%<@KHIa@|S1i`5R)0Ik zKwj}Q>~;r|Kna)!XLe8p^}bT!2-g z*?yX~fK#oBclP5?A3j}l?|Zjb-GBSu`N!^t>EFhQ`?V6k*GRM01bmGI>uSJDQmene zRP}X|y^XJFjWq8(M@~Q{sY^m)DY78*k-_}id=-T1M;x_jDkKg<$Hndxl&US0TJ_$e z9kSNVPMALaSOQpl9dg-J68(&4GVlkOJ6ONu>3kt_>JSgq-Ym-ky51Y0uYb#{f032G zSw{UnuYI4_UIJCIo2`bL-_)uR_oQq@7I) zi`rth%aQwQ4d3~-hQgV2$$y20MTYf@1AR;V>`N1SvrU)y+E*n9+s=y4(q6A}UF7YP z75U#w)Y0Z7ZbU(aHhT3$-KT#dXcMw3y)w%QdsR~Y5}msCbmYQRiwEuV>`|$F_qPJ} zw*vO%Pg1J$fB6Wz&Q>0QT#^5Oa(H@JeE;$EV4wfHlTuI`YSA0!X@4dCaBH0WZ=QJ{ zFduRW!n!q%G4G1WvJ^eGN>%wYK^F0Ur4#5zbNPXr1LR zI_Q^r{Z|bsV^{uC4;Oj-v6&b*ngJi_1{09=x2WzDq5s#<$g3aBJgJ{NWj28Bw*f-e z$0S%AtI2J);mY6Luzz8WTevyN6GHEr(acX60y&%*hbn-QXw(xqNGm24de&r_`{TEyrORX8oFqd6i zOU=%gyh^#wl?0`2Vn1A2<-JCKC2X-|-m4bxtE$@gQvaVp&3~?;BDSn$UxlG*{=7*M z(T1b2>($pO)SYW5#oxQuP1|Ac{{>2^FcE7jq^;^BwIY?PmHA`ECg8n0r4;{JgB463 z^Y*V56SD-#$D#DBk}+I&SaJ|GOwF z&i^~z2+W_E|9=j*fN!Muv-9!2sQl~&&z&fH3sdaX{Ps$<{*M!JA3;FZ@as@2S0VXdw<=xou5!=b0pNKmO2~k*||SR!V&obj1wC6(5&0QQI!4ku>dX_9&neDNT|G{-(v>ooli$-C@8}dK*z&V zq?xA?;sAQcV5!C|hu`Y%wK|RG?(;U~$&~*Y_G93iRRD|SzuW2VADHsLx4V`9CrKp0 z6X-P%8etBX5P#7g3S8C-h|4IaW6^SHAWMBZlC=TE)PLwyw zQOfC(OaG5;<6m#&<*C~Vw4GAYmaVVD{7>s*pDn!J4?E=v)Lfam7d0Sw9 z!`m(m$A4rJGjJm6IW7$aCLyr4gy$$=PQ=Iz3rL4_-fpP;i`e&vnwlQ^^%|F6iR~g^{8VwF?MqD^@T^ff%j<3rA3+N%iVn8p@-(^?i-)~S-7hzqQqp!sG zWuJ8u4~;29a~ikM5QH3#G4?qiGM1U>D-Km=%zr`tW{7?bTM{1)1*5(XtS4vM(W01T zIvk@HND_E2sL9xcMq|}uLh6Ihn zkP0k+DW1Tdr~dPOB&L8O>hVTIy`#KV58d5Y!T5Lt{vrrxs)BBe90wi8T+qO1M?yhS z0Dl5|Sm1`MsuUU`^a9=d!W|q%Rl7&hbOtj}RuV=Rj8I0z{Mg4_!H1AbV@ge?Bnt7E z&BzQ815RV+0^(E3#H!ry($IxSDoy0|=BU*KR~bdIoAn_?!y}9M_Mz z0Oq)i2;yR2VjqGV%i4$UOK@p2XlnEWe9ECJwlnOMOBMw4H zk;YQ!6}V#!O=1|$5!~V+@&WbaZ=*^N-AUL-{446lyWfu@qd-F zn@mrLDdE}pxHR@XDNr8ac;tg8l|W87@+io07081KLJ!dpeHxyo%^OH^EF&(zpiK18 z&R-99_v|a*6tFL*d9|H}S{{7jV%|gQR~(*)pRnYIo>oW_5}ioK_IqewHPGzJHB8%@ zE|CXB@bs6YFi;E7ey8(^B)JU!6Mq9Q)^ej*PIXXewv)E9bgCX%_mRkQE2tjF&LeS{ ztM&I9Iy@c|DYJDv9lBHSVjr3-!A~S0@^J{?9gFh_G!3y*`f&gsVSePg!1>>an4WqN3dIhokO%nobBJe{_)-P+P}inc_E*y@yA`DYp@jO* zr=xMq;eyfI0_ssITR|it7ufgH23fE2%9-CnTB^IMh=cfqZ_gtL-xD7OaZqR$7zNG0 z>W!P8*hKMujyx>jpPG3}vtpsZ{hNmYDB0vHM$M5m#uazYAfGmxgz z1+ai{t__#Y+qZA48uUqZTJ8SinbqjykgTQMtU|sLcQ?j>Ma)#W=Ir;w4~)i9x=%`3 zKo@=hp+#^gu-t3t(uTGhdnE(8<#N(enjzVVYa3J=K}Qn^rRI@T#((MWVLTp_+a7A3 z4~Nba{6Lnf_8;i*oQ9Gk!DtV8Fd{hIdj5X6WcvRE z1d;OHv*SnSkCgriBaFYzGq=^WH{ zT5VgConouexT46Nf_t+BnBM0KEMoo!MLu8-a*NSxBznXZK60EewQvsa?ojIzRJ~Sa z_n`ZGNsYI5|5FxQUd(LASEd-_@pJK?HIjx1s% z6l2u<6L6MgMfx8RcySD$h3m?NNQ*}ii{1d zHYV@xP%7J{xoIJ5BemZOUER=Bru8Ii?`a*qet}L|NPj}tkG;Cy)KKQqAfll%eGAUk zYP|?E;n4Aedmi8rPr!3V^K#!)mv>70IGy>U?9AK_FgyR2&?-&7q5s5Glt%Ubz9B95 z7N52i8``>%@Mo(?P9jM~FSRa4g8mZ+{(>4A;?m4&v5#Wk&rWH^l;!SSj_mUsPk&n^;>rn(YQ%6jfraf*c1R!A}20S$*B8tGYC)%%(A zQc{%oFzZbG21-^F?R2TG7k;)*d_y4#JqQKblYb3)1nSf=T~K<7!%w)=vjRlrv#X&m z6@#Q{p;O9fRf4hz4niq0s$S}q%38|tKC0wuj~8L!oy3fUlf<)3!pZ3*q*-?V7Tj3H zWffYMon2SJ0$whb3R8_m|2ATfcp{A&DEE^~bI?*+bcqkBI1R0J60efAOk$W3CSvS6 zxqrh5DPJ!U$JrW_w@wv(#2~^9oDm5zBf!(;Ws3O)17%vZo8?8Lr8p*!e9-)rCLIKW znvxaEt&*S^bbntht3d48>j*rN8&JIh`5acoy%v1E*yZ3|8U$F1=&!H%^wk^m%5h)4 zK}|P8hJtVZrRBj0wM7uMoBt|6m?=-?Cw~>PP@YLA@n~x|xCkO+p7m^rAaXpy8Yw-t zc}=M_z=s?{0nEsK8BUlXgMnf`2uV15PwfcP%9e*C8 zrfkzhKmTkr6H};4wHMt?kV7<|MA*H-6W~r3E+%=&Q;hWIKO-Co6sr@IX^+He&WPO7G-a%E$s_oRyJx6`+ry0pN{@<{r>cG0)Unst8BA7uGU=GWS0 zjY;Nnt{ib+(U`Z!#0Mw9w}&t9iX>Vl$jFJ4!!vyjDYe95vkk&+>jH1oa%nhj8m&%F zhez+u`q!T?PL8hnC+`kl-kDiTM!~FZ41nat*RPQPE|4=uFYl^!-mf@L;D1gjYK~w` znQ|(G6B17B5PN1n5puDx*R9S6c#_p$@%F{#=RyDC=<097!{%>_^~*UyD9~?`$A6Pd z{#(lBNY|abS`Xp-;bC*|ewhD|oSd;Vl0_?n)?VH@`Y9_m}5`tN!4`m`7!|su+=wNDBnJg3O?4 zg=8+OHJ}PZqBdEj7>AY--k^rbhRj?h*Ft7QOw=N?E+=bYtOR1z;(shIF>9d;R>Na1 z5^LcyqC^a2MpQ?d%!mh_Y~f=m7&G`mr%DJ!8Bn(b#*C9JHHsLS5g#Vt72;zoVTy(P zi_?=*UMavL177xnQr<%Bi^oF4cC=?d-9ndjFp#Vs=d@ z;1d7e!NI{>!~fUqbhqb!Pm&VQLB1-)2ck1#nNTF1Yqd}zzSuH^>I z1DfsN({hbsHK*ZnLT#j0Pq=cD4|Kyd9FMEevg*M7QfZR=hJUj=6X|h^ajKM;FMwA# zjVXgJTwS)roE zTR8DDdst~*MqU|L%gSwMk3yBaUHLZCz?AfIaRFKmVI4c5GGs;B%ν-?|;dJ+;@ z8k=OI>ZZO(wtsPCx@@{FR!m{YDy8OmZyiMilP1SivU46lquuG6AI}u8Q?9T@E#L6i z>}GWl?CQ4CsJ-R}c5^t*O)X}ViqE|1%no0nk_+c6U15QVh3(VMkx=bAQrx>QFh^R@r|i{j+bo+Q~K4 z7N_pbA|JuB$;q#(mgi;FGmZ30bDk3S19fje6pl4eX}^aot7FOFNGclS7S_$~PLber z(9&3RN8dzVK~zi8>zkF-1)!@!Q?9(GoNfx^DpFdO$2x}8HPvin@?v-oWd(gpd9tik>zYL{}qx-Y!-pgwT1ab|Tams|z0Ykvs6 zh>}nozI=T=JiYD@PA<+*2UoWBD7h4t2}`G?{`Kr=3s=bOn6)E9PI-}Voty4zY|E#L zX>(!@%)t`~g{8G|cg7AVhv0A(sA&OYT(_yANPper)3>o59Y5!o-6{->^`76uAm?lR zir^P4|LVG+13@I_C44pDt#=O%$Ws4RI5e|O^?j|F%bEdDeY2IxZ7#w7 z34i!_J_MJB9`B*uPTB9N7r3mrTP}`(5$f3+*PDU=`}wkeeSFzJ=?|_>kIt0u-)0oP zXEdnB5ownEgrkozzl5(7f>*2}A>$CF@*Ju4N{eLq%(5h;x-t{wCHCaO8KRJs_kwr~)b!RBB zd!uByX*Mhq*dKg3y*wX$>JP5etX&*k{jJ8F2jLc8tY|9|xx}LoxBks(SfOzF0ik_~ zd{UF&Y;1JY3MgmA&CUwv{sLP~QpUO8c7X{~J2AlCdFanA>}{P18?nlv)6&;HQGas` z9%cAxNi6_(q8oD(O$5;mX}j1FEasxE_Oh-pX5`@mvxW&Qd3LGgTYQmeHUCWVgv7>u zGgi1(A4ytseV@!Dajj+^uj+NN`fQ%U6Z6WM;?VQK(F)8VVa&k+#wa^f_r_m6ue&83 z`yYL;nUh*0ZkA)=1@tMySiSCWSAUgPtBPNE#=L0ZSa#mL*2-a5#Y<_d>Sc?pt+6_6 zjGX#aU@5RAOTw~)9wWq}c&cK)QB`GGv$6FAtw6!X&VN@p|4$6d^)CP|yZ^txbFjVt z^GwNcoYIj$*LU_?H^1n+i|GxY^-jYv zJrC8wlG$g5NG7kcv@B1uDB{Mv^hnEly)(>@4AVv7jEupZyFPrP9%RqOT*xuwA`MS; zP(PYN01W#-VxUh=T0#C*;M-#!1YRuvJKav#y#N1pe|H=I^CZcEd;w-`2=%&WA+uNETQ(hNlw5qWNk4fSTC57Tjb6Oyr~b6esBq^eS(oIsVwu9E!77J%6Z_aAt=)E4#E5 z@O1+x>|YUBf!{1|gKn1VmLjNfg00D#9QZx3Kb#wLuXCkZqVcmyQ!eBPi|OIZ*C&^! zU;3BVm;Ddd!}H^h*B3`uf2%?$^VovpX1L7_o-|o+cOEMl%d|& zJkZ|%Zm#sg#WPEH?SHTe%%}aQ>1E#g`|Q?wt?R``#CLJU=Z#Abg46B8uA( zvL^^3)9c)(g2&1?cZ)5vtF9N9=l}TULLl0|XTYk^YH#l^EBo={^j3*<(|;SM6=K}V9614zp4)PP z1uuduJEzM}g4YnLo|sprDHk}192dJ&kgB#yYTaw}cF0OMJ5l~LeFdQTcKxEEBj)A@tPbQ*io1<{-j ze0AheoB#3(zv6Qx7J*)$#I-1>k>^Jb)OG%y&vnSE^?z$Dc{k=UF?|cP(#pe8dax+_ zU{#qtYSq^^sBjxpxK024sXqRTjj-!%fC&8ckGf$V)hf;u;s*PhYrHuTp*2hBJ=)$Nm)iSYG9Jm$0LhKjcAyLn}tRFwh zVqe=@j(^<5swhw1L%%kYm#qs%-LK6ki{ngew4~Au*hp2FfM~u!O`mY}zxs@n{6mkE z{L7OT2dH*yAas39f|arA*ftxka_p`Rb5g_gG3P7xbHzTv3_`KS$R)jbgPKtty5-$+ zIg!TBMmzK-Q;~&RhwKV{3EE@6;#BA zwd|uXwDso=O+;(-!>1p%R zDSxHpvt~<}JmKws)eOuAIQe1Y^Z#J2U5jB>;q4##HRh*JtM30Fa|vw8{=d7sw`1P_ z+}k_Y?*E@6E!qD+=|Z65Nq@v8;F%PEaXfw$m0!Fn^d!pO!W3IIzqwSe|C5Qwdg34B z30`*pzw-V6oxM(XcdP%OB2~_t$_8G(5=E z|A$+GOXYvxxc|Siv%kN!|DGg0oBZd7#8&dRX`8lbo1P*4UjP6A|Nl-Dst5q2008d% BzKZ|= diff --git a/charts/latest/blob-csi-driver/templates/blobfuse-proxy.yaml b/charts/latest/blob-csi-driver/templates/blobfuse-proxy.yaml deleted file mode 100644 index c131d8316..000000000 --- a/charts/latest/blob-csi-driver/templates/blobfuse-proxy.yaml +++ /dev/null @@ -1,126 +0,0 @@ -{{- if .Values.node.enableBlobfuseProxy -}} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: csi-blobfuse-proxy - namespace: {{ .Release.Namespace }} - labels: - {{- include "blob.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: csi-blobfuse-proxy - template: - metadata: - labels: - app: csi-blobfuse-proxy - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: type - operator: NotIn - values: - - virtual-kubelet - initContainers: - - name: prepare-binaries -{{- if hasPrefix "/" .Values.image.blob.repository }} - image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" -{{- else }} - image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" -{{- end }} - command: ['sh', '-c', "cp /blobfuse-proxy/*.deb /tmp/"] - volumeMounts: - - mountPath: /tmp - name: tmp-dir - containers: - - command: - - nsenter - - '--target' - - '1' - - '--mount' - - '--uts' - - '--ipc' - - '--net' - - '--pid' - - '--' - - sh - - '-c' - - | - set -xe - INSTALL_BLOBFUSE=${INSTALL_BLOBFUSE:-true} - if (( "${INSTALL_BLOBFUSE}" == "true" )) - then - dpkg -i /tmp/packages-microsoft-prod.deb && apt update && apt-get install -y blobfuse=${BLOBFUSE_VERSION} - fi - dpkg -i /tmp/blobfuse-proxy.deb - mkdir -p /var/lib/kubelet/plugins/blob.csi.azure.com - systemctl enable blobfuse-proxy - systemctl start blobfuse-proxy - - SET_MAX_FILE_NUM=${SET_MAX_OPEN_FILE_NUM:-true} - if (( "${SET_MAX_OPEN_FILE_NUM}" == "true" )) - then - sysctl -w fs.file-max=${MAX_FILE_NUM} - fi - - updateDBConfigPath="/etc/updatedb.conf" - DISABLE_UPDATEDB=${DISABLE_UPDATEDB:-true} - if (( "${DISABLE_UPDATEDB}" == "true" )) && (( test -f ${updateDBConfigPath} )) - then - echo "before changing ${updateDBConfigPath}:" - cat ${updateDBConfigPath} - sed -i 's/PRUNEPATHS="\/tmp/PRUNEPATHS="\/mnt \/var\/lib\/kubelet \/tmp/g' ${updateDBConfigPath} - sed -i 's/PRUNEFS="NFS/PRUNEFS="fuse blobfuse NFS/g' ${updateDBConfigPath} - echo "after change:" - cat ${updateDBConfigPath} - fi - - # "waiting for blobfuse-proxy service to start" - sleep 3s - # tail blobfuse proxy logs - journalctl -u blobfuse-proxy -f -{{- if hasPrefix "/" .Values.image.blob.repository }} - image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" -{{- else }} - image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" -{{- end }} - imagePullPolicy: IfNotPresent - name: sysctl-install-blobfuse-proxy - env: - - name: DEBIAN_FRONTEND - value: "noninteractive" - - name: INSTALL_BLOBFUSE - value: "{{ .Values.node.blobfuseProxy.installBlobfuse }}" - - name: BLOBFUSE_VERSION - value: "{{ .Values.node.blobfuseProxy.blobfuseVersion }}" - - name: SET_MAX_OPEN_FILE_NUM - value: "{{ .Values.node.blobfuseProxy.setMaxOpenFileNum }}" - - name: MAX_FILE_NUM - value: "{{ .Values.node.blobfuseProxy.maxOpenFileNum }}" - - name: DISABLE_UPDATEDB - value: "{{ .Values.node.blobfuseProxy.disableUpdateDB }}" - resources: - limits: - memory: 200Mi - requests: - cpu: 10m - memory: 1Mi - securityContext: - privileged: true - hostNetwork: true - hostPID: true - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-node-critical - restartPolicy: Always - tolerations: - - operator: Exists - volumes: - - hostPath: - path: /tmp - type: DirectoryOrCreate - name: tmp-dir -{{ end }} diff --git a/charts/latest/blob-csi-driver/templates/csi-blob-controller.yaml b/charts/latest/blob-csi-driver/templates/csi-blob-controller.yaml index 077181daa..529877581 100644 --- a/charts/latest/blob-csi-driver/templates/csi-blob-controller.yaml +++ b/charts/latest/blob-csi-driver/templates/csi-blob-controller.yaml @@ -40,6 +40,9 @@ spec: {{- if .Values.controller.runOnMaster}} node-role.kubernetes.io/master: "" {{- end}} + {{- if .Values.controller.runOnControlPlane}} + node-role.kubernetes.io/control-plane: "" + {{- end}} {{- with .Values.controller.nodeSelector }} {{ toYaml . | indent 8 }} {{- end }} diff --git a/charts/latest/blob-csi-driver/templates/csi-blob-node.yaml b/charts/latest/blob-csi-driver/templates/csi-blob-node.yaml index 3ceea89a2..eeb1db2f4 100644 --- a/charts/latest/blob-csi-driver/templates/csi-blob-node.yaml +++ b/charts/latest/blob-csi-driver/templates/csi-blob-node.yaml @@ -32,6 +32,9 @@ spec: imagePullSecrets: {{ toYaml .Values.imagePullSecrets | indent 8 }} {{- end }} +{{- if .Values.node.enableBlobfuseProxy }} + hostPID: true +{{- end }} hostNetwork: true dnsPolicy: Default serviceAccountName: {{ .Values.serviceAccount.node }} @@ -56,6 +59,38 @@ spec: {{- with .Values.node.tolerations }} tolerations: {{ toYaml . | indent 8 }} +{{- end }} +{{- if .Values.node.enableBlobfuseProxy }} + initContainers: + - name: install-blobfuse-proxy +{{- if hasPrefix "/" .Values.image.blob.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- else }} + image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}" +{{- end }} + imagePullPolicy: IfNotPresent + command: + - "/blobfuse-proxy/init.sh" + securityContext: + privileged: true + env: + - name: DEBIAN_FRONTEND + value: "noninteractive" + - name: INSTALL_BLOBFUSE + value: "{{ .Values.node.blobfuseProxy.installBlobfuse }}" + - name: BLOBFUSE_VERSION + value: "{{ .Values.node.blobfuseProxy.blobfuseVersion }}" + - name: SET_MAX_OPEN_FILE_NUM + value: "{{ .Values.node.blobfuseProxy.setMaxOpenFileNum }}" + - name: MAX_FILE_NUM + value: "{{ .Values.node.blobfuseProxy.maxOpenFileNum }}" + - name: DISABLE_UPDATEDB + value: "{{ .Values.node.blobfuseProxy.disableUpdateDB }}" + volumeMounts: + - name: host-usr + mountPath: /host/usr + - name: host-etc + mountPath: /host/etc {{- end }} containers: - name: liveness-probe @@ -188,6 +223,14 @@ spec: {{- end }} resources: {{- toYaml .Values.node.resources.blob | nindent 12 }} volumes: +{{- if .Values.node.enableBlobfuseProxy }} + - name: host-usr + hostPath: + path: /usr + - name: host-etc + hostPath: + path: /etc +{{- end }} - hostPath: path: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }} type: DirectoryOrCreate diff --git a/charts/latest/blob-csi-driver/values.yaml b/charts/latest/blob-csi-driver/values.yaml index bc1505940..960a510fd 100644 --- a/charts/latest/blob-csi-driver/values.yaml +++ b/charts/latest/blob-csi-driver/values.yaml @@ -64,6 +64,7 @@ controller: healthPort: 29632 replicas: 2 runOnMaster: false + runOnControlPlane: false logLevel: 5 resources: csiProvisioner: @@ -117,7 +118,7 @@ node: enableBlobfuseProxy: false blobfuseProxy: installBlobfuse: true - blobfuseVersion: 1.4.3 + blobfuseVersion: 1.4.4 setMaxOpenFileNum: true maxOpenFileNum: "9000000" disableUpdateDB: true @@ -147,8 +148,6 @@ node: nodeSelector: {} tolerations: - operator: "Exists" - livenessProbe: - healthPort: 29633 feature: enableFSGroupPolicy: false diff --git a/deploy/blobfuse-proxy.yaml b/deploy/blobfuse-proxy.yaml deleted file mode 100644 index 3bb533ac9..000000000 --- a/deploy/blobfuse-proxy.yaml +++ /dev/null @@ -1,115 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: csi-blobfuse-proxy - namespace: kube-system -spec: - selector: - matchLabels: - app: csi-blobfuse-proxy - template: - metadata: - labels: - app: csi-blobfuse-proxy - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: type - operator: NotIn - values: - - virtual-kubelet - initContainers: - - name: prepare-binaries - image: mcr.microsoft.com/k8s/csi/blob-csi:latest - command: ['sh', '-c', "cp /blobfuse-proxy/*.deb /tmp/"] - volumeMounts: - - mountPath: /tmp - name: tmp-dir - containers: - - command: - - nsenter - - '--target' - - '1' - - '--mount' - - '--uts' - - '--ipc' - - '--net' - - '--pid' - - '--' - - sh - - '-c' - - | - set -xe - INSTALL_BLOBFUSE=${INSTALL_BLOBFUSE:-true} - if (( "${INSTALL_BLOBFUSE}" == "true" )) - then - dpkg -i /tmp/packages-microsoft-prod.deb && apt update && apt-get install -y blobfuse=${BLOBFUSE_VERSION} - fi - dpkg -i /tmp/blobfuse-proxy.deb - mkdir -p /var/lib/kubelet/plugins/blob.csi.azure.com - systemctl enable blobfuse-proxy - systemctl start blobfuse-proxy - - SET_MAX_FILE_NUM=${SET_MAX_OPEN_FILE_NUM:-true} - if (( "${SET_MAX_OPEN_FILE_NUM}" == "true" )) - then - sysctl -w fs.file-max=${MAX_FILE_NUM} - fi - - updateDBConfigPath="/etc/updatedb.conf" - DISABLE_UPDATEDB=${DISABLE_UPDATEDB:-true} - if (( "${DISABLE_UPDATEDB}" == "true" )) && (( test -f ${updateDBConfigPath} )) - then - echo "before changing ${updateDBConfigPath}:" - cat ${updateDBConfigPath} - sed -i 's/PRUNEPATHS="\/tmp/PRUNEPATHS="\/mnt \/var\/lib\/kubelet \/tmp/g' ${updateDBConfigPath} - sed -i 's/PRUNEFS="NFS/PRUNEFS="fuse blobfuse NFS/g' ${updateDBConfigPath} - echo "after change:" - cat ${updateDBConfigPath} - fi - - # "waiting for blobfuse-proxy service to start" - sleep 3s - # tail blobfuse proxy logs - journalctl -u blobfuse-proxy -f - image: mcr.microsoft.com/k8s/csi/blob-csi:latest - imagePullPolicy: IfNotPresent - name: sysctl-install-blobfuse-proxy - env: - - name: DEBIAN_FRONTEND - value: "noninteractive" - - name: INSTALL_BLOBFUSE - value: "true" - - name: BLOBFUSE_VERSION - value: 1.4.3 - - name: SET_MAX_OPEN_FILE_NUM - value: "true" - - name: MAX_FILE_NUM - value: "9000000" - - name: DISABLE_UPDATEDB - value: "true" - resources: - limits: - memory: 200Mi - requests: - cpu: 10m - memory: 1Mi - securityContext: - privileged: true - hostNetwork: true - hostPID: true - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-node-critical - restartPolicy: Always - tolerations: - - operator: Exists - volumes: - - hostPath: - path: /tmp - type: DirectoryOrCreate - name: tmp-dir diff --git a/deploy/csi-blob-node.yaml b/deploy/csi-blob-node.yaml index cd6eb0cb1..60bf9cb25 100644 --- a/deploy/csi-blob-node.yaml +++ b/deploy/csi-blob-node.yaml @@ -18,6 +18,7 @@ spec: app: csi-blob-node spec: hostNetwork: true + hostPID: true dnsPolicy: Default serviceAccountName: csi-blob-node-sa nodeSelector: @@ -34,6 +35,32 @@ spec: priorityClassName: system-node-critical tolerations: - operator: "Exists" + initContainers: + - name: install-blobfuse-proxy + image: mcr.microsoft.com/k8s/csi/blob-csi:latest + imagePullPolicy: IfNotPresent + command: + - "/blobfuse-proxy/init.sh" + securityContext: + privileged: true + env: + - name: DEBIAN_FRONTEND + value: "noninteractive" + - name: INSTALL_BLOBFUSE + value: "true" + - name: BLOBFUSE_VERSION + value: 1.4.4 + - name: SET_MAX_OPEN_FILE_NUM + value: "true" + - name: MAX_FILE_NUM + value: "9000000" + - name: DISABLE_UPDATEDB + value: "true" + volumeMounts: + - name: host-usr + mountPath: /host/usr + - name: host-etc + mountPath: /host/etc containers: - name: liveness-probe volumeMounts: @@ -139,6 +166,12 @@ spec: cpu: 10m memory: 20Mi volumes: + - name: host-usr + hostPath: + path: /usr + - name: host-etc + hostPath: + path: /etc - hostPath: path: /var/lib/kubelet/plugins/blob.csi.azure.com type: DirectoryOrCreate diff --git a/deploy/install-driver.sh b/deploy/install-driver.sh index 6d809bb26..f21560ca5 100755 --- a/deploy/install-driver.sh +++ b/deploy/install-driver.sh @@ -42,7 +42,6 @@ kubectl apply -f $repo/csi-blob-controller.yaml if [[ "$#" -gt 1 ]]; then if [[ "$2" == *"blobfuse-proxy"* ]]; then echo "set enable-blobfuse-proxy as true ..." - kubectl apply -f $repo/blobfuse-proxy.yaml if [[ "$2" == *"local"* ]]; then cat $repo/csi-blob-node.yaml | sed 's/enable-blobfuse-proxy=false/enable-blobfuse-proxy=true/g' | kubectl apply -f - else diff --git a/deploy/uninstall-driver.sh b/deploy/uninstall-driver.sh index 01550e9e9..f194ea600 100755 --- a/deploy/uninstall-driver.sh +++ b/deploy/uninstall-driver.sh @@ -33,12 +33,6 @@ if [ $ver != "master" ]; then repo="$repo/$ver" fi -if [[ "$#" -gt 1 ]]; then - if [[ "$2" == *"blobfuse-proxy"* ]]; then - echo "remove blobfuse-proxy deployment ..." - kubectl delete -f $repo/blobfuse-proxy.yaml - fi -fi echo "Uninstalling Azure Blob Storage CSI driver, version: $ver ..." kubectl delete -f $repo/csi-blob-controller.yaml --ignore-not-found diff --git a/docs/driver-parameters.md b/docs/driver-parameters.md index 283b5b5c2..26b107335 100644 --- a/docs/driver-parameters.md +++ b/docs/driver-parameters.md @@ -22,13 +22,16 @@ tags | [tags](https://docs.microsoft.com/en-us/azure/azure-resource-manager/mana matchTags | whether matching tags when driver tries to find a suitable storage account | `true`,`false` | No | `false` useDataPlaneAPI | specify whether use data plane API for blob container create/delete, this could solve the SRP API throltting issue since data plane API has almost no limit, while it would fail when there is firewall or vnet setting on storage account | `true`,`false` | No | `false` --- | **Following parameters are only for blobfuse** | --- | --- | -subscriptionID | specify Azure subscription ID in which blob storage directory will be created | Azure subscription ID | No | if not empty, `resourceGroup` must be provided +subscriptionID | specify Azure subscription ID in which blob storage directory will be created, cross subscription is only supported when `useDataPlaneAPI` is set as `true` | Azure subscription ID | No | if not empty, `resourceGroup` must be provided storeAccountKey | whether store account key to k8s secret

Note:
`false` means driver would leverage kubelet identity to get account key | `true`,`false` | No | `true` secretName | specify secret name to store account key | | No | secretNamespace | specify the namespace of secret to store account key | `default`,`kube-system`, etc | No | pvc namespace isHnsEnabled | enable `Hierarchical namespace` for Azure DataLake storage account | `true`,`false` | No | `false` --- | **Following parameters are only for NFS protocol** | --- | --- | mountPermissions | mounted folder permissions. The default is `0777`, if set as `0`, driver will not perform `chmod` after mount | `0777` | No | +vnetResourceGroup | specify vnet resource group where virtual network is | existing resource group name | No | if empty, driver will use the `vnetResourceGroup` value in azure cloud config file +vnetName | virtual network name | existing virtual network name | No | if empty, driver will use the `vnetName` value in azure cloud config file +subnetName | subnet name | existing subnet name of the agent node | No | if empty, driver will use the `subnetName` value in azure cloud config file - `fsGroup` securityContext setting @@ -70,10 +73,6 @@ nodeStageSecretRef.name | secret name that stores(check below examples):
`azu nodeStageSecretRef.namespace | secret namespace | k8s namespace | Yes | --- | **Following parameters are only for NFS protocol** | --- | --- | volumeAttributes.mountPermissions | mounted folder permissions | `0777` | No | ---- | **Following parameters are only for NFS vnet setting** | --- | --- | -vnetResourceGroup | specify vnet resource group where virtual network is | existing resource group name | No | if empty, driver will use the `vnetResourceGroup` value in azure cloud config file -vnetName | virtual network name | existing virtual network name | No | if empty, driver will use the `vnetName` value in azure cloud config file -subnetName | subnet name | existing subnet name of the agent node | No | if empty, driver will use the `subnetName` value in azure cloud config file --- | **Following parameters are only for feature: blobfuse [Managed Identity and Service Principal Name auth](https://github.com/Azure/azure-storage-fuse#environment-variables)** | --- | --- | volumeAttributes.AzureStorageAuthType | Authentication Type | `Key`, `SAS`, `MSI`, `SPN` | No | `Key` volumeAttributes.AzureStorageIdentityClientID | Identity Client ID | | No | diff --git a/docs/install-csi-driver-master.md b/docs/install-csi-driver-master.md index 6125f4eee..dff7e4b31 100644 --- a/docs/install-csi-driver-master.md +++ b/docs/install-csi-driver-master.md @@ -35,7 +35,7 @@ csi-blob-node-dr4s4 3/3 Running 0 35s ### clean up Blob CSI driver - Option#1. remote uninstall ```console -curl -skSL https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/deploy/uninstall-driver.sh | bash -s master blobfuse-proxy -- +curl -skSL https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/master/deploy/uninstall-driver.sh | bash -s master -- ``` - Option#2. local uninstall @@ -43,5 +43,5 @@ curl -skSL https://raw.githubusercontent.com/kubernetes-sigs/blob-csi-driver/mas git clone https://github.com/kubernetes-sigs/blob-csi-driver.git cd blob-csi-driver git checkout master -./deploy/install-driver.sh master local +./deploy/uninstall-driver.sh master local ``` diff --git a/pkg/blob/blob.go b/pkg/blob/blob.go index f6e7b5b78..69c096243 100644 --- a/pkg/blob/blob.go +++ b/pkg/blob/blob.go @@ -803,3 +803,18 @@ func createStorageAccountSecret(account, key string) map[string]string { secret[defaultSecretAccountKey] = key return secret } + +// setKeyValueInMap set key/value pair in map +// key in the map is case insensitive, if key already exists, overwrite existing value +func setKeyValueInMap(m map[string]string, key, value string) { + if m == nil { + return + } + for k := range m { + if strings.EqualFold(k, key) { + m[k] = value + return + } + } + m[key] = value +} diff --git a/pkg/blob/blob_test.go b/pkg/blob/blob_test.go index fa6c17cdd..f03331553 100644 --- a/pkg/blob/blob_test.go +++ b/pkg/blob/blob_test.go @@ -950,3 +950,57 @@ func TestCreateStorageAccountSecret(t *testing.T) { t.Errorf("Expected account name(%s), Actual account name(%s); Expected account key(%s), Actual account key(%s)", "TestAccountName", result[defaultSecretAccountName], "TestAccountKey", result[defaultSecretAccountKey]) } } + +func TestSetKeyValueInMap(t *testing.T) { + tests := []struct { + desc string + m map[string]string + key string + value string + expected map[string]string + }{ + { + desc: "nil map", + key: "key", + value: "value", + }, + { + desc: "empty map", + m: map[string]string{}, + key: "key", + value: "value", + expected: map[string]string{"key": "value"}, + }, + { + desc: "non-empty map", + m: map[string]string{"k": "v"}, + key: "key", + value: "value", + expected: map[string]string{ + "k": "v", + "key": "value", + }, + }, + { + desc: "same key already exists", + m: map[string]string{"subDir": "value2"}, + key: "subDir", + value: "value", + expected: map[string]string{"subDir": "value"}, + }, + { + desc: "case insentive key already exists", + m: map[string]string{"subDir": "value2"}, + key: "subdir", + value: "value", + expected: map[string]string{"subDir": "value"}, + }, + } + + for _, test := range tests { + setKeyValueInMap(test.m, test.key, test.value) + if !reflect.DeepEqual(test.m, test.expected) { + t.Errorf("test[%s]: unexpected output: %v, expected result: %v", test.desc, test.m, test.expected) + } + } +} diff --git a/pkg/blob/controllerserver.go b/pkg/blob/controllerserver.go index d48ab3260..9a3ada247 100644 --- a/pkg/blob/controllerserver.go +++ b/pkg/blob/controllerserver.go @@ -296,7 +296,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) validContainerName = containerNamePrefix + "-" + volName } validContainerName = getValidContainerName(validContainerName, protocol) - parameters[containerNameField] = validContainerName + setKeyValueInMap(parameters, containerNameField, validContainerName) } var volumeID string @@ -343,7 +343,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) isOperationSucceeded = true // reset secretNamespace field in VolumeContext - parameters[secretNamespaceField] = secretNamespace + setKeyValueInMap(parameters, secretNamespaceField, secretNamespace) return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ VolumeId: volumeID, diff --git a/pkg/blob/nodeserver.go b/pkg/blob/nodeserver.go index c607603c6..de1e66359 100644 --- a/pkg/blob/nodeserver.go +++ b/pkg/blob/nodeserver.go @@ -76,11 +76,11 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu context := req.GetVolumeContext() if context != nil { if strings.EqualFold(context[ephemeralField], trueValue) { - context[secretNamespaceField] = context[podNamespaceField] + setKeyValueInMap(context, secretNamespaceField, context[podNamespaceField]) if !d.allowInlineVolumeKeyAccessWithIdentity { // only get storage account from secret - context[getAccountKeyFromSecretField] = trueValue - context[storageAccountField] = "" + setKeyValueInMap(context, getAccountKeyFromSecretField, trueValue) + setKeyValueInMap(context, storageAccountField, "") } klog.V(2).Infof("NodePublishVolume: ephemeral volume(%s) mount on %s, VolumeContext: %v", volumeID, target, context) _, err := d.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{ diff --git a/pkg/blobfuse-proxy/debpackage/etc/systemd/system/blobfuse-proxy.service b/pkg/blobfuse-proxy/blobfuse-proxy.service similarity index 100% rename from pkg/blobfuse-proxy/debpackage/etc/systemd/system/blobfuse-proxy.service rename to pkg/blobfuse-proxy/blobfuse-proxy.service diff --git a/pkg/blobfuse-proxy/debpackage/DEBIAN/control b/pkg/blobfuse-proxy/debpackage/DEBIAN/control deleted file mode 100644 index 8d0142775..000000000 --- a/pkg/blobfuse-proxy/debpackage/DEBIAN/control +++ /dev/null @@ -1,7 +0,0 @@ -Package: blobfuse-proxy -Version: 0.1.1 -Section: base -Priority: optional -Architecture: all -Maintainer: Blob CSI driver maintainers -Description: blobfuse proxy diff --git a/pkg/blobfuse-proxy/init.sh b/pkg/blobfuse-proxy/init.sh new file mode 100644 index 000000000..f859b97ab --- /dev/null +++ b/pkg/blobfuse-proxy/init.sh @@ -0,0 +1,67 @@ +#!/bin/sh + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +INSTALL_BLOBFUSE=${INSTALL_BLOBFUSE:-true} +DISABLE_UPDATEDB=${DISABLE_UPDATEDB:-true} +SET_MAX_OPEN_FILE_NUM=${SET_MAX_OPEN_FILE_NUM:-true} + +HOST_CMD="nsenter --mount=/proc/1/ns/mnt" + +cp /blobfuse-proxy/packages-microsoft-prod.deb /host/etc/ + +# install/update blobfuse +if [ "${INSTALL_BLOBFUSE}" = "true" ] +then + $HOST_CMD dpkg -i /etc/packages-microsoft-prod.deb && \ + $HOST_CMD apt update && \ + $HOST_CMD apt-get install -y blobfuse="${BLOBFUSE_VERSION}" +fi + +if [ ! -f "/host/usr/bin/blobfuse-proxy" ];then + echo "copy blobfuse-proxy...." + cp /blobfuse-proxy/blobfuse-proxy /host/usr/bin/blobfuse-proxy + chmod 755 /host/usr/bin/blobfuse-proxy +fi + +if [ ! -f "/host/usr/lib/systemd/system/blobfuse-proxy.service" ];then + echo "copy blobfuse-proxy.service...." + mkdir -p /host/usr/lib/systemd/system + cp /blobfuse-proxy/blobfuse-proxy.service /host/usr/lib/systemd/system/blobfuse-proxy.service +fi + +$HOST_CMD systemctl daemon-reload +$HOST_CMD systemctl enable blobfuse-proxy.service +# According to the issue https://github.com/kubernetes-sigs/blob-csi-driver/issues/693, +# do NOT RESTART blobfuse-proxy, just start it at first time. +$HOST_CMD systemctl start blobfuse-proxy.service + +if [ "${SET_MAX_OPEN_FILE_NUM}" = "true" ] +then + $HOST_CMD sysctl -w fs.file-max="${MAX_FILE_NUM}" +fi + +updateDBConfigPath="/host/etc/updatedb.conf" +if [ "${DISABLE_UPDATEDB}" = "true" ] && [ -f ${updateDBConfigPath} ] +then + echo "before changing ${updateDBConfigPath}:" + cat ${updateDBConfigPath} + sed -i 's/PRUNEPATHS="\/tmp/PRUNEPATHS="\/mnt \/var\/lib\/kubelet \/tmp/g' ${updateDBConfigPath} + sed -i 's/PRUNEFS="NFS/PRUNEFS="fuse blobfuse NFS/g' ${updateDBConfigPath} + echo "after change:" + cat ${updateDBConfigPath} +fi diff --git a/pkg/blobfuse-proxy/rpmbuild/SPECS/utils.spec b/pkg/blobfuse-proxy/rpmbuild/SPECS/utils.spec deleted file mode 100644 index 45dd49ef4..000000000 --- a/pkg/blobfuse-proxy/rpmbuild/SPECS/utils.spec +++ /dev/null @@ -1,25 +0,0 @@ -############################################################################### -# Spec file for blobfuse-proxy -################################################################################ -# Configured to be built by non-root user -################################################################################ -# -Summary: Utility scripts for creating RPM package for blobfuse-proxy -Name: blobfuse-proxy -Version: v0.1.0 -Release: 1 -License: Apache -Group: System -Packager: David Both -Requires: bash -BuildRoot: ~/rpmbuild/ - -%description -Utility scripts for creating RPM package for blobfuse-proxy - -%install -mkdir -p %{buildroot}/usr/bin/ -cp blobfuse-proxy %{buildroot}/usr/bin/blobfuse-proxy - -%files -/usr/bin/blobfuse-proxy diff --git a/pkg/blobplugin/Dockerfile b/pkg/blobplugin/Dockerfile index 4b5214729..9571f22bd 100644 --- a/pkg/blobplugin/Dockerfile +++ b/pkg/blobplugin/Dockerfile @@ -18,10 +18,18 @@ ARG ARCH=amd64 ARG binary=./_output/${ARCH}/blobplugin COPY ${binary} /blobplugin +RUN mkdir /blobfuse-proxy/ + +COPY ./pkg/blobfuse-proxy/init.sh /blobfuse-proxy/ +COPY ./pkg/blobfuse-proxy/blobfuse-proxy.service /blobfuse-proxy/ +COPY ./_output/${ARCH}/blobfuse-proxy /blobfuse-proxy/ + +RUN chmod +x /blobfuse-proxy/init.sh && \ + chmod +x /blobfuse-proxy/blobfuse-proxy.service && \ + chmod +x /blobfuse-proxy/blobfuse-proxy + RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates uuid-dev util-linux mount udev wget e2fsprogs nfs-common netbase -RUN mkdir /blobfuse-proxy/ -COPY ./_output/blobfuse-proxy.deb /blobfuse-proxy/ ARG ARCH=amd64 RUN if [ "$ARCH" = "amd64" ] ; then \ clean-install libcurl4-gnutls-dev && \